Upgrade V8 to version 4.9.385.28
https://chromium.googlesource.com/v8/v8/+/4.9.385.28
FPIIM-449
Change-Id: I4b2e74289d4bf3667f2f3dc8aa2e541f63e26eb4
diff --git a/src/heap/incremental-marking.cc b/src/heap/incremental-marking.cc
index aadd17c..52d0ca4 100644
--- a/src/heap/incremental-marking.cc
+++ b/src/heap/incremental-marking.cc
@@ -2,44 +2,73 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-#include "src/v8.h"
-
#include "src/heap/incremental-marking.h"
#include "src/code-stubs.h"
#include "src/compilation-cache.h"
#include "src/conversions.h"
+#include "src/heap/gc-idle-time-handler.h"
+#include "src/heap/gc-tracer.h"
+#include "src/heap/mark-compact-inl.h"
#include "src/heap/objects-visiting.h"
#include "src/heap/objects-visiting-inl.h"
+#include "src/v8.h"
namespace v8 {
namespace internal {
+IncrementalMarking::StepActions IncrementalMarking::IdleStepActions() {
+ return StepActions(IncrementalMarking::NO_GC_VIA_STACK_GUARD,
+ IncrementalMarking::FORCE_MARKING,
+ IncrementalMarking::DO_NOT_FORCE_COMPLETION);
+}
+
IncrementalMarking::IncrementalMarking(Heap* heap)
: heap_(heap),
+ observer_(*this, kAllocatedThreshold),
state_(STOPPED),
+ is_compacting_(false),
steps_count_(0),
old_generation_space_available_at_start_of_incremental_(0),
old_generation_space_used_at_start_of_incremental_(0),
+ bytes_rescanned_(0),
should_hurry_(false),
marking_speed_(0),
+ bytes_scanned_(0),
allocated_(0),
+ write_barriers_invoked_since_last_step_(0),
idle_marking_delay_counter_(0),
no_marking_scope_depth_(0),
unscanned_bytes_of_large_object_(0),
- was_activated_(false) {}
+ was_activated_(false),
+ finalize_marking_completed_(false),
+ incremental_marking_finalization_rounds_(0),
+ request_type_(COMPLETE_MARKING) {}
+
+
+bool IncrementalMarking::BaseRecordWrite(HeapObject* obj, Object* value) {
+ HeapObject* value_heap_obj = HeapObject::cast(value);
+ MarkBit value_bit = Marking::MarkBitFrom(value_heap_obj);
+ DCHECK(!Marking::IsImpossible(value_bit));
+
+ MarkBit obj_bit = Marking::MarkBitFrom(obj);
+ DCHECK(!Marking::IsImpossible(obj_bit));
+ bool is_black = Marking::IsBlack(obj_bit);
+
+ if (is_black && Marking::IsWhite(value_bit)) {
+ WhiteToGreyAndPush(value_heap_obj, value_bit);
+ RestartIfNotMarking();
+ }
+ return is_compacting_ && is_black;
+}
void IncrementalMarking::RecordWriteSlow(HeapObject* obj, Object** slot,
Object* value) {
- if (BaseRecordWrite(obj, slot, value) && slot != NULL) {
- MarkBit obj_bit = Marking::MarkBitFrom(obj);
- if (Marking::IsBlack(obj_bit)) {
- // Object is not going to be rescanned we need to record the slot.
- heap_->mark_compact_collector()->RecordSlot(HeapObject::RawField(obj, 0),
- slot, value);
- }
+ if (BaseRecordWrite(obj, value) && slot != NULL) {
+ // Object is not going to be rescanned we need to record the slot.
+ heap_->mark_compact_collector()->RecordSlot(obj, slot, value);
}
}
@@ -66,7 +95,7 @@
void IncrementalMarking::RecordCodeTargetPatch(Code* host, Address pc,
HeapObject* value) {
if (IsMarking()) {
- RelocInfo rinfo(pc, RelocInfo::CODE_TARGET, 0, host);
+ RelocInfo rinfo(heap_->isolate(), pc, RelocInfo::CODE_TARGET, 0, host);
RecordWriteIntoCode(host, &rinfo, value);
}
}
@@ -77,7 +106,7 @@
Code* host = heap_->isolate()
->inner_pointer_to_code_cache()
->GcSafeFindCodeForInnerPointer(pc);
- RelocInfo rinfo(pc, RelocInfo::CODE_TARGET, 0, host);
+ RelocInfo rinfo(heap_->isolate(), pc, RelocInfo::CODE_TARGET, 0, host);
RecordWriteIntoCode(host, &rinfo, value);
}
}
@@ -86,10 +115,10 @@
void IncrementalMarking::RecordWriteOfCodeEntrySlow(JSFunction* host,
Object** slot,
Code* value) {
- if (BaseRecordWrite(host, slot, value)) {
+ if (BaseRecordWrite(host, value)) {
DCHECK(slot != NULL);
heap_->mark_compact_collector()->RecordCodeEntrySlot(
- reinterpret_cast<Address>(slot), value);
+ host, reinterpret_cast<Address>(slot), value);
}
}
@@ -97,58 +126,84 @@
void IncrementalMarking::RecordWriteIntoCodeSlow(HeapObject* obj,
RelocInfo* rinfo,
Object* value) {
- MarkBit value_bit = Marking::MarkBitFrom(HeapObject::cast(value));
- if (Marking::IsWhite(value_bit)) {
- MarkBit obj_bit = Marking::MarkBitFrom(obj);
- if (Marking::IsBlack(obj_bit)) {
- BlackToGreyAndUnshift(obj, obj_bit);
- RestartIfNotMarking();
- }
- // Object is either grey or white. It will be scanned if survives.
- return;
- }
-
- if (is_compacting_) {
- MarkBit obj_bit = Marking::MarkBitFrom(obj);
- if (Marking::IsBlack(obj_bit)) {
+ if (BaseRecordWrite(obj, value)) {
// Object is not going to be rescanned. We need to record the slot.
heap_->mark_compact_collector()->RecordRelocSlot(rinfo,
Code::cast(value));
+ }
+}
+
+
+void IncrementalMarking::RecordWrites(HeapObject* obj) {
+ if (IsMarking()) {
+ MarkBit obj_bit = Marking::MarkBitFrom(obj);
+ if (Marking::IsBlack(obj_bit)) {
+ MemoryChunk* chunk = MemoryChunk::FromAddress(obj->address());
+ if (chunk->IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR)) {
+ chunk->set_progress_bar(0);
+ }
+ BlackToGreyAndUnshift(obj, obj_bit);
+ RestartIfNotMarking();
}
}
}
+void IncrementalMarking::BlackToGreyAndUnshift(HeapObject* obj,
+ MarkBit mark_bit) {
+ DCHECK(Marking::MarkBitFrom(obj) == mark_bit);
+ DCHECK(obj->Size() >= 2 * kPointerSize);
+ DCHECK(IsMarking());
+ Marking::BlackToGrey(mark_bit);
+ int obj_size = obj->Size();
+ MemoryChunk::IncrementLiveBytesFromGC(obj, -obj_size);
+ bytes_scanned_ -= obj_size;
+ int64_t old_bytes_rescanned = bytes_rescanned_;
+ bytes_rescanned_ = old_bytes_rescanned + obj_size;
+ if ((bytes_rescanned_ >> 20) != (old_bytes_rescanned >> 20)) {
+ if (bytes_rescanned_ > 2 * heap_->PromotedSpaceSizeOfObjects()) {
+ // If we have queued twice the heap size for rescanning then we are
+ // going around in circles, scanning the same objects again and again
+ // as the program mutates the heap faster than we can incrementally
+ // trace it. In this case we switch to non-incremental marking in
+ // order to finish off this marking phase.
+ if (FLAG_trace_incremental_marking) {
+ PrintIsolate(
+ heap()->isolate(),
+ "Hurrying incremental marking because of lack of progress\n");
+ }
+ marking_speed_ = kMaxMarkingSpeed;
+ }
+ }
+
+ heap_->mark_compact_collector()->marking_deque()->Unshift(obj);
+}
+
+
+void IncrementalMarking::WhiteToGreyAndPush(HeapObject* obj, MarkBit mark_bit) {
+ Marking::WhiteToGrey(mark_bit);
+ heap_->mark_compact_collector()->marking_deque()->Push(obj);
+}
+
+
static void MarkObjectGreyDoNotEnqueue(Object* obj) {
if (obj->IsHeapObject()) {
HeapObject* heap_obj = HeapObject::cast(obj);
MarkBit mark_bit = Marking::MarkBitFrom(HeapObject::cast(obj));
if (Marking::IsBlack(mark_bit)) {
- MemoryChunk::IncrementLiveBytesFromGC(heap_obj->address(),
- -heap_obj->Size());
+ MemoryChunk::IncrementLiveBytesFromGC(heap_obj, -heap_obj->Size());
}
Marking::AnyToGrey(mark_bit);
}
}
-static inline void MarkBlackOrKeepGrey(HeapObject* heap_object,
- MarkBit mark_bit, int size) {
- DCHECK(!Marking::IsImpossible(mark_bit));
- if (mark_bit.Get()) return;
- mark_bit.Set();
- MemoryChunk::IncrementLiveBytesFromGC(heap_object->address(), size);
- DCHECK(Marking::IsBlack(mark_bit));
-}
-
-
static inline void MarkBlackOrKeepBlack(HeapObject* heap_object,
MarkBit mark_bit, int size) {
DCHECK(!Marking::IsImpossible(mark_bit));
if (Marking::IsBlack(mark_bit)) return;
Marking::MarkBlack(mark_bit);
- MemoryChunk::IncrementLiveBytesFromGC(heap_object->address(), size);
- DCHECK(Marking::IsBlack(mark_bit));
+ MemoryChunk::IncrementLiveBytesFromGC(heap_object, size);
}
@@ -186,9 +241,8 @@
int already_scanned_offset = start_offset;
bool scan_until_end = false;
do {
- VisitPointersWithAnchor(heap, HeapObject::RawField(object, 0),
- HeapObject::RawField(object, start_offset),
- HeapObject::RawField(object, end_offset));
+ VisitPointers(heap, object, HeapObject::RawField(object, start_offset),
+ HeapObject::RawField(object, end_offset));
start_offset = end_offset;
end_offset = Min(object_size, end_offset + kProgressBarScanningChunk);
scan_until_end =
@@ -196,7 +250,12 @@
} while (scan_until_end && start_offset < object_size);
chunk->set_progress_bar(start_offset);
if (start_offset < object_size) {
- heap->mark_compact_collector()->marking_deque()->UnshiftGrey(object);
+ if (Marking::IsGrey(Marking::MarkBitFrom(object))) {
+ heap->mark_compact_collector()->marking_deque()->Unshift(object);
+ } else {
+ DCHECK(Marking::IsBlack(Marking::MarkBitFrom(object)));
+ heap->mark_compact_collector()->UnshiftBlack(object);
+ }
heap->incremental_marking()->NotifyIncompleteScanOfObject(
object_size - (start_offset - already_scanned_offset));
}
@@ -218,44 +277,28 @@
VisitNativeContext(map, context);
}
- INLINE(static void VisitPointer(Heap* heap, Object** p)) {
- Object* obj = *p;
- if (obj->IsHeapObject()) {
- heap->mark_compact_collector()->RecordSlot(p, p, obj);
- MarkObject(heap, obj);
+ INLINE(static void VisitPointer(Heap* heap, HeapObject* object, Object** p)) {
+ Object* target = *p;
+ if (target->IsHeapObject()) {
+ heap->mark_compact_collector()->RecordSlot(object, p, target);
+ MarkObject(heap, target);
}
}
- INLINE(static void VisitPointers(Heap* heap, Object** start, Object** end)) {
+ INLINE(static void VisitPointers(Heap* heap, HeapObject* object,
+ Object** start, Object** end)) {
for (Object** p = start; p < end; p++) {
- Object* obj = *p;
- if (obj->IsHeapObject()) {
- heap->mark_compact_collector()->RecordSlot(start, p, obj);
- MarkObject(heap, obj);
- }
- }
- }
-
- INLINE(static void VisitPointersWithAnchor(Heap* heap, Object** anchor,
- Object** start, Object** end)) {
- for (Object** p = start; p < end; p++) {
- Object* obj = *p;
- if (obj->IsHeapObject()) {
- heap->mark_compact_collector()->RecordSlot(anchor, p, obj);
- MarkObject(heap, obj);
+ Object* target = *p;
+ if (target->IsHeapObject()) {
+ heap->mark_compact_collector()->RecordSlot(object, p, target);
+ MarkObject(heap, target);
}
}
}
// Marks the object grey and pushes it on the marking stack.
INLINE(static void MarkObject(Heap* heap, Object* obj)) {
- HeapObject* heap_object = HeapObject::cast(obj);
- MarkBit mark_bit = Marking::MarkBitFrom(heap_object);
- if (mark_bit.data_only()) {
- MarkBlackOrKeepGrey(heap_object, mark_bit, heap_object->Size());
- } else if (Marking::IsWhite(mark_bit)) {
- heap->incremental_marking()->WhiteToGreyAndPush(heap_object, mark_bit);
- }
+ IncrementalMarking::MarkObject(heap, HeapObject::cast(obj));
}
// Marks the object black without pushing it on the marking stack.
@@ -264,9 +307,8 @@
HeapObject* heap_object = HeapObject::cast(obj);
MarkBit mark_bit = Marking::MarkBitFrom(heap_object);
if (Marking::IsWhite(mark_bit)) {
- mark_bit.Set();
- MemoryChunk::IncrementLiveBytesFromGC(heap_object->address(),
- heap_object->Size());
+ Marking::MarkBlack(mark_bit);
+ MemoryChunk::IncrementLiveBytesFromGC(heap_object, heap_object->Size());
return true;
}
return false;
@@ -278,11 +320,11 @@
public:
explicit IncrementalMarkingRootMarkingVisitor(
IncrementalMarking* incremental_marking)
- : incremental_marking_(incremental_marking) {}
+ : heap_(incremental_marking->heap()) {}
- void VisitPointer(Object** p) { MarkObjectByPointer(p); }
+ void VisitPointer(Object** p) override { MarkObjectByPointer(p); }
- void VisitPointers(Object** start, Object** end) {
+ void VisitPointers(Object** start, Object** end) override {
for (Object** p = start; p < end; p++) MarkObjectByPointer(p);
}
@@ -291,18 +333,10 @@
Object* obj = *p;
if (!obj->IsHeapObject()) return;
- HeapObject* heap_object = HeapObject::cast(obj);
- MarkBit mark_bit = Marking::MarkBitFrom(heap_object);
- if (mark_bit.data_only()) {
- MarkBlackOrKeepGrey(heap_object, mark_bit, heap_object->Size());
- } else {
- if (Marking::IsWhite(mark_bit)) {
- incremental_marking_->WhiteToGreyAndPush(heap_object, mark_bit);
- }
- }
+ IncrementalMarking::MarkObject(heap_, HeapObject::cast(obj));
}
- IncrementalMarking* incremental_marking_;
+ Heap* heap_;
};
@@ -317,17 +351,6 @@
if (is_marking) {
chunk->SetFlag(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING);
chunk->SetFlag(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING);
-
- // It's difficult to filter out slots recorded for large objects.
- if (chunk->owner()->identity() == LO_SPACE &&
- chunk->size() > static_cast<size_t>(Page::kPageSize) && is_compacting) {
- chunk->SetFlag(MemoryChunk::RESCAN_ON_EVACUATION);
- }
- } else if (chunk->owner()->identity() == CELL_SPACE ||
- chunk->owner()->identity() == PROPERTY_CELL_SPACE ||
- chunk->scan_on_scavenge()) {
- chunk->ClearFlag(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING);
- chunk->ClearFlag(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING);
} else {
chunk->ClearFlag(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING);
chunk->SetFlag(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING);
@@ -335,7 +358,7 @@
}
-void IncrementalMarking::SetNewSpacePageFlags(NewSpacePage* chunk,
+void IncrementalMarking::SetNewSpacePageFlags(MemoryChunk* chunk,
bool is_marking) {
chunk->SetFlag(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING);
if (is_marking) {
@@ -368,10 +391,7 @@
void IncrementalMarking::DeactivateIncrementalWriteBarrier() {
- DeactivateIncrementalWriteBarrierForSpace(heap_->old_pointer_space());
- DeactivateIncrementalWriteBarrierForSpace(heap_->old_data_space());
- DeactivateIncrementalWriteBarrierForSpace(heap_->cell_space());
- DeactivateIncrementalWriteBarrierForSpace(heap_->property_cell_space());
+ DeactivateIncrementalWriteBarrierForSpace(heap_->old_space());
DeactivateIncrementalWriteBarrierForSpace(heap_->map_space());
DeactivateIncrementalWriteBarrierForSpace(heap_->code_space());
DeactivateIncrementalWriteBarrierForSpace(heap_->new_space());
@@ -403,10 +423,7 @@
void IncrementalMarking::ActivateIncrementalWriteBarrier() {
- ActivateIncrementalWriteBarrier(heap_->old_pointer_space());
- ActivateIncrementalWriteBarrier(heap_->old_data_space());
- ActivateIncrementalWriteBarrier(heap_->cell_space());
- ActivateIncrementalWriteBarrier(heap_->property_cell_space());
+ ActivateIncrementalWriteBarrier(heap_->old_space());
ActivateIncrementalWriteBarrier(heap_->map_space());
ActivateIncrementalWriteBarrier(heap_->code_space());
ActivateIncrementalWriteBarrier(heap_->new_space());
@@ -419,15 +436,17 @@
}
-bool IncrementalMarking::ShouldActivate() {
- return WorthActivating() && heap_->NextGCIsLikelyToBeFull();
+bool IncrementalMarking::ShouldActivateEvenWithoutIdleNotification() {
+ return CanBeActivated() &&
+ heap_->HeapIsFullEnoughToStartIncrementalMarking(
+ heap_->old_generation_allocation_limit());
}
bool IncrementalMarking::WasActivated() { return was_activated_; }
-bool IncrementalMarking::WorthActivating() {
+bool IncrementalMarking::CanBeActivated() {
#ifndef DEBUG
static const intptr_t kActivationThreshold = 8 * MB;
#else
@@ -438,8 +457,8 @@
// Only start incremental marking in a safe state: 1) when incremental
// marking is turned on, 2) when we are currently not in a GC, and
// 3) when we are currently not serializing or deserializing the heap.
- return FLAG_incremental_marking && FLAG_incremental_marking_steps &&
- heap_->gc_state() == Heap::NOT_IN_GC &&
+ // Don't switch on for very small heaps.
+ return FLAG_incremental_marking && heap_->gc_state() == Heap::NOT_IN_GC &&
heap_->deserialization_complete() &&
!heap_->isolate()->serializer_enabled() &&
heap_->PromotedSpaceSizeOfObjects() > kActivationThreshold;
@@ -461,6 +480,21 @@
}
+void IncrementalMarking::NotifyOfHighPromotionRate() {
+ if (IsMarking()) {
+ if (marking_speed_ < kFastMarking) {
+ if (FLAG_trace_gc) {
+ PrintIsolate(heap()->isolate(),
+ "Increasing marking speed to %d "
+ "due to high promotion rate\n",
+ static_cast<int>(kFastMarking));
+ }
+ marking_speed_ = kFastMarking;
+ }
+ }
+}
+
+
static void PatchIncrementalMarkingRecordWriteStubs(
Heap* heap, RecordWriteStub::Mode mode) {
UnseededNumberDictionary* stubs = heap->code_stubs();
@@ -482,22 +516,24 @@
}
-void IncrementalMarking::Start(CompactionFlag flag) {
+void IncrementalMarking::Start(const char* reason) {
if (FLAG_trace_incremental_marking) {
- PrintF("[IncrementalMarking] Start\n");
+ PrintF("[IncrementalMarking] Start (%s)\n",
+ (reason == nullptr) ? "unknown reason" : reason);
}
DCHECK(FLAG_incremental_marking);
- DCHECK(FLAG_incremental_marking_steps);
DCHECK(state_ == STOPPED);
DCHECK(heap_->gc_state() == Heap::NOT_IN_GC);
DCHECK(!heap_->isolate()->serializer_enabled());
+ HistogramTimerScope incremental_marking_scope(
+ heap_->isolate()->counters()->gc_incremental_marking_start());
ResetStepCounters();
was_activated_ = true;
if (!heap_->mark_compact_collector()->sweeping_in_progress()) {
- StartMarking(flag);
+ StartMarking();
} else {
if (FLAG_trace_incremental_marking) {
PrintF("[IncrementalMarking] Start sweeping.\n");
@@ -505,16 +541,18 @@
state_ = SWEEPING;
}
- heap_->new_space()->LowerInlineAllocationLimit(kAllocatedThreshold);
+ heap_->new_space()->AddInlineAllocationObserver(&observer_);
+
+ incremental_marking_job()->Start(heap_);
}
-void IncrementalMarking::StartMarking(CompactionFlag flag) {
+void IncrementalMarking::StartMarking() {
if (FLAG_trace_incremental_marking) {
PrintF("[IncrementalMarking] Start marking\n");
}
- is_compacting_ = !FLAG_never_compact && (flag == ALLOW_COMPACTION) &&
+ is_compacting_ = !FLAG_never_compact &&
heap_->mark_compact_collector()->StartCompaction(
MarkCompactCollector::INCREMENTAL_COMPACTION);
@@ -526,7 +564,8 @@
PatchIncrementalMarkingRecordWriteStubs(heap_, mode);
- heap_->mark_compact_collector()->EnsureMarkingDequeIsCommittedAndInitialize();
+ heap_->mark_compact_collector()->EnsureMarkingDequeIsCommittedAndInitialize(
+ MarkCompactCollector::kMaxMarkingDequeSize);
ActivateIncrementalWriteBarrier();
@@ -550,8 +589,6 @@
IncrementalMarkingRootMarkingVisitor visitor(this);
heap_->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG);
- heap_->mark_compact_collector()->MarkWeakObjectToCodeTable();
-
// Ready to start incremental marking.
if (FLAG_trace_incremental_marking) {
PrintF("[IncrementalMarking] Running\n");
@@ -559,12 +596,176 @@
}
-void IncrementalMarking::PrepareForScavenge() {
- if (!IsMarking()) return;
- NewSpacePageIterator it(heap_->new_space()->FromSpaceStart(),
- heap_->new_space()->FromSpaceEnd());
- while (it.has_next()) {
- Bitmap::Clear(it.next());
+void IncrementalMarking::MarkRoots() {
+ DCHECK(!finalize_marking_completed_);
+ DCHECK(IsMarking());
+
+ IncrementalMarkingRootMarkingVisitor visitor(this);
+ heap_->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG);
+}
+
+
+void IncrementalMarking::MarkObjectGroups() {
+ DCHECK(!finalize_marking_completed_);
+ DCHECK(IsMarking());
+
+ IncrementalMarkingRootMarkingVisitor visitor(this);
+ heap_->mark_compact_collector()->MarkImplicitRefGroups(&MarkObject);
+ heap_->isolate()->global_handles()->IterateObjectGroups(
+ &visitor, &MarkCompactCollector::IsUnmarkedHeapObjectWithHeap);
+ heap_->isolate()->global_handles()->RemoveImplicitRefGroups();
+ heap_->isolate()->global_handles()->RemoveObjectGroups();
+}
+
+
+void IncrementalMarking::ProcessWeakCells() {
+ DCHECK(!finalize_marking_completed_);
+ DCHECK(IsMarking());
+
+ Object* the_hole_value = heap()->the_hole_value();
+ Object* weak_cell_obj = heap()->encountered_weak_cells();
+ Object* weak_cell_head = Smi::FromInt(0);
+ WeakCell* prev_weak_cell_obj = NULL;
+ while (weak_cell_obj != Smi::FromInt(0)) {
+ WeakCell* weak_cell = reinterpret_cast<WeakCell*>(weak_cell_obj);
+ // We do not insert cleared weak cells into the list, so the value
+ // cannot be a Smi here.
+ HeapObject* value = HeapObject::cast(weak_cell->value());
+ // Remove weak cells with live objects from the list, they do not need
+ // clearing.
+ if (MarkCompactCollector::IsMarked(value)) {
+ // Record slot, if value is pointing to an evacuation candidate.
+ Object** slot = HeapObject::RawField(weak_cell, WeakCell::kValueOffset);
+ heap_->mark_compact_collector()->RecordSlot(weak_cell, slot, *slot);
+ // Remove entry somewhere after top.
+ if (prev_weak_cell_obj != NULL) {
+ prev_weak_cell_obj->set_next(weak_cell->next());
+ }
+ weak_cell_obj = weak_cell->next();
+ weak_cell->clear_next(the_hole_value);
+ } else {
+ if (weak_cell_head == Smi::FromInt(0)) {
+ weak_cell_head = weak_cell;
+ }
+ prev_weak_cell_obj = weak_cell;
+ weak_cell_obj = weak_cell->next();
+ }
+ }
+ // Top may have changed.
+ heap()->set_encountered_weak_cells(weak_cell_head);
+}
+
+
+bool ShouldRetainMap(Map* map, int age) {
+ if (age == 0) {
+ // The map has aged. Do not retain this map.
+ return false;
+ }
+ Object* constructor = map->GetConstructor();
+ if (!constructor->IsHeapObject() ||
+ Marking::IsWhite(Marking::MarkBitFrom(HeapObject::cast(constructor)))) {
+ // The constructor is dead, no new objects with this map can
+ // be created. Do not retain this map.
+ return false;
+ }
+ return true;
+}
+
+
+void IncrementalMarking::RetainMaps() {
+ // Do not retain dead maps if flag disables it or there is
+ // - memory pressure (reduce_memory_footprint_),
+ // - GC is requested by tests or dev-tools (abort_incremental_marking_).
+ bool map_retaining_is_disabled = heap()->ShouldReduceMemory() ||
+ heap()->ShouldAbortIncrementalMarking() ||
+ FLAG_retain_maps_for_n_gc == 0;
+ ArrayList* retained_maps = heap()->retained_maps();
+ int length = retained_maps->Length();
+ // The number_of_disposed_maps separates maps in the retained_maps
+ // array that were created before and after context disposal.
+ // We do not age and retain disposed maps to avoid memory leaks.
+ int number_of_disposed_maps = heap()->number_of_disposed_maps_;
+ for (int i = 0; i < length; i += 2) {
+ DCHECK(retained_maps->Get(i)->IsWeakCell());
+ WeakCell* cell = WeakCell::cast(retained_maps->Get(i));
+ if (cell->cleared()) continue;
+ int age = Smi::cast(retained_maps->Get(i + 1))->value();
+ int new_age;
+ Map* map = Map::cast(cell->value());
+ MarkBit map_mark = Marking::MarkBitFrom(map);
+ if (i >= number_of_disposed_maps && !map_retaining_is_disabled &&
+ Marking::IsWhite(map_mark)) {
+ if (ShouldRetainMap(map, age)) {
+ MarkObject(heap(), map);
+ }
+ Object* prototype = map->prototype();
+ if (age > 0 && prototype->IsHeapObject() &&
+ Marking::IsWhite(Marking::MarkBitFrom(HeapObject::cast(prototype)))) {
+ // The prototype is not marked, age the map.
+ new_age = age - 1;
+ } else {
+ // The prototype and the constructor are marked, this map keeps only
+ // transition tree alive, not JSObjects. Do not age the map.
+ new_age = age;
+ }
+ } else {
+ new_age = FLAG_retain_maps_for_n_gc;
+ }
+ // Compact the array and update the age.
+ if (new_age != age) {
+ retained_maps->Set(i + 1, Smi::FromInt(new_age));
+ }
+ }
+}
+
+
+void IncrementalMarking::FinalizeIncrementally() {
+ DCHECK(!finalize_marking_completed_);
+ DCHECK(IsMarking());
+
+ double start = heap_->MonotonicallyIncreasingTimeInMs();
+
+ int old_marking_deque_top =
+ heap_->mark_compact_collector()->marking_deque()->top();
+
+ // After finishing incremental marking, we try to discover all unmarked
+ // objects to reduce the marking load in the final pause.
+ // 1) We scan and mark the roots again to find all changes to the root set.
+ // 2) We mark the object groups.
+ // 3) Age and retain maps embedded in optimized code.
+ // 4) Remove weak cell with live values from the list of weak cells, they
+ // do not need processing during GC.
+ MarkRoots();
+ MarkObjectGroups();
+ if (incremental_marking_finalization_rounds_ == 0) {
+ // Map retaining is needed for perfromance, not correctness,
+ // so we can do it only once at the beginning of the finalization.
+ RetainMaps();
+ }
+ ProcessWeakCells();
+
+ int marking_progress =
+ abs(old_marking_deque_top -
+ heap_->mark_compact_collector()->marking_deque()->top());
+
+ double end = heap_->MonotonicallyIncreasingTimeInMs();
+ double delta = end - start;
+ heap_->tracer()->AddMarkingTime(delta);
+ heap_->tracer()->AddIncrementalMarkingFinalizationStep(delta);
+ if (FLAG_trace_incremental_marking) {
+ PrintF(
+ "[IncrementalMarking] Finalize incrementally round %d, "
+ "spent %d ms, marking progress %d.\n",
+ static_cast<int>(delta), incremental_marking_finalization_rounds_,
+ marking_progress);
+ }
+
+ ++incremental_marking_finalization_rounds_;
+ if ((incremental_marking_finalization_rounds_ >=
+ FLAG_max_incremental_marking_finalization_rounds) ||
+ (marking_progress <
+ FLAG_min_progress_during_incremental_marking_finalization)) {
+ finalize_marking_completed_ = true;
}
}
@@ -620,10 +821,7 @@
void IncrementalMarking::VisitObject(Map* map, HeapObject* obj, int size) {
- MarkBit map_mark_bit = Marking::MarkBitFrom(map);
- if (Marking::IsWhite(map_mark_bit)) {
- WhiteToGreyAndPush(map, map_mark_bit);
- }
+ MarkObject(heap_, map);
IncrementalMarkingMarkingVisitor::IterateBody(map, obj);
@@ -639,6 +837,14 @@
}
+void IncrementalMarking::MarkObject(Heap* heap, HeapObject* obj) {
+ MarkBit mark_bit = Marking::MarkBitFrom(obj);
+ if (Marking::IsWhite(mark_bit)) {
+ heap->incremental_marking()->WhiteToGreyAndPush(obj, mark_bit);
+ }
+}
+
+
intptr_t IncrementalMarking::ProcessMarkingDeque(intptr_t bytes_to_process) {
intptr_t bytes_processed = 0;
Map* filler_map = heap_->one_pointer_filler_map();
@@ -655,10 +861,7 @@
int size = obj->SizeFromMap(map);
unscanned_bytes_of_large_object_ = 0;
VisitObject(map, obj, size);
- int delta = (size - unscanned_bytes_of_large_object_);
- // TODO(jochen): remove after http://crbug.com/381820 is resolved.
- CHECK_LT(0, delta);
- bytes_processed += delta;
+ bytes_processed += size - unscanned_bytes_of_large_object_;
}
return bytes_processed;
}
@@ -685,7 +888,7 @@
if (state() == MARKING) {
double start = 0.0;
if (FLAG_trace_incremental_marking || FLAG_print_cumulative_gc_stat) {
- start = base::OS::TimeCurrentMillis();
+ start = heap_->MonotonicallyIncreasingTimeInMs();
if (FLAG_trace_incremental_marking) {
PrintF("[IncrementalMarking] Hurry\n");
}
@@ -695,7 +898,7 @@
ProcessMarkingDeque();
state_ = COMPLETE;
if (FLAG_trace_incremental_marking || FLAG_print_cumulative_gc_stat) {
- double end = base::OS::TimeCurrentMillis();
+ double end = heap_->MonotonicallyIncreasingTimeInMs();
double delta = end - start;
heap_->tracer()->AddMarkingTime(delta);
if (FLAG_trace_incremental_marking) {
@@ -708,7 +911,7 @@
if (FLAG_cleanup_code_caches_at_gc) {
PolymorphicCodeCache* poly_cache = heap_->polymorphic_code_cache();
Marking::GreyToBlack(Marking::MarkBitFrom(poly_cache));
- MemoryChunk::IncrementLiveBytesFromGC(poly_cache->address(),
+ MemoryChunk::IncrementLiveBytesFromGC(poly_cache,
PolymorphicCodeCache::kSize);
}
@@ -722,7 +925,7 @@
MarkBit mark_bit = Marking::MarkBitFrom(cache);
if (Marking::IsGrey(mark_bit)) {
Marking::GreyToBlack(mark_bit);
- MemoryChunk::IncrementLiveBytesFromGC(cache->address(), cache->Size());
+ MemoryChunk::IncrementLiveBytesFromGC(cache, cache->Size());
}
}
context = Context::cast(context)->get(Context::NEXT_CONTEXT_LINK);
@@ -730,12 +933,13 @@
}
-void IncrementalMarking::Abort() {
+void IncrementalMarking::Stop() {
if (IsStopped()) return;
if (FLAG_trace_incremental_marking) {
- PrintF("[IncrementalMarking] Aborting.\n");
+ PrintF("[IncrementalMarking] Stopping.\n");
}
- heap_->new_space()->LowerInlineAllocationLimit(0);
+
+ heap_->new_space()->RemoveInlineAllocationObserver(&observer_);
IncrementalMarking::set_should_hurry(false);
ResetStepCounters();
if (IsMarking()) {
@@ -763,7 +967,8 @@
Hurry();
state_ = STOPPED;
is_compacting_ = false;
- heap_->new_space()->LowerInlineAllocationLimit(0);
+
+ heap_->new_space()->RemoveInlineAllocationObserver(&observer_);
IncrementalMarking::set_should_hurry(false);
ResetStepCounters();
PatchIncrementalMarkingRecordWriteStubs(heap_,
@@ -774,6 +979,20 @@
}
+void IncrementalMarking::FinalizeMarking(CompletionAction action) {
+ DCHECK(!finalize_marking_completed_);
+ if (FLAG_trace_incremental_marking) {
+ PrintF(
+ "[IncrementalMarking] requesting finalization of incremental "
+ "marking.\n");
+ }
+ request_type_ = FINALIZATION;
+ if (action == GC_VIA_STACK_GUARD) {
+ heap_->isolate()->stack_guard()->RequestGC();
+ }
+}
+
+
void IncrementalMarking::MarkingComplete(CompletionAction action) {
state_ = COMPLETE;
// We will set the stack guard to request a GC now. This will mean the rest
@@ -785,20 +1004,52 @@
if (FLAG_trace_incremental_marking) {
PrintF("[IncrementalMarking] Complete (normal).\n");
}
+ request_type_ = COMPLETE_MARKING;
if (action == GC_VIA_STACK_GUARD) {
heap_->isolate()->stack_guard()->RequestGC();
}
}
-void IncrementalMarking::Epilogue() { was_activated_ = false; }
+void IncrementalMarking::Epilogue() {
+ was_activated_ = false;
+ finalize_marking_completed_ = false;
+ incremental_marking_finalization_rounds_ = 0;
+}
+
+
+double IncrementalMarking::AdvanceIncrementalMarking(
+ intptr_t step_size_in_bytes, double deadline_in_ms,
+ IncrementalMarking::StepActions step_actions) {
+ DCHECK(!IsStopped());
+
+ if (step_size_in_bytes == 0) {
+ step_size_in_bytes = GCIdleTimeHandler::EstimateMarkingStepSize(
+ static_cast<size_t>(GCIdleTimeHandler::kIncrementalMarkingStepTimeInMs),
+ static_cast<size_t>(
+ heap()
+ ->tracer()
+ ->FinalIncrementalMarkCompactSpeedInBytesPerMillisecond()));
+ }
+
+ double remaining_time_in_ms = 0.0;
+ do {
+ Step(step_size_in_bytes, step_actions.completion_action,
+ step_actions.force_marking, step_actions.force_completion);
+ remaining_time_in_ms =
+ deadline_in_ms - heap()->MonotonicallyIncreasingTimeInMs();
+ } while (remaining_time_in_ms >=
+ 2.0 * GCIdleTimeHandler::kIncrementalMarkingStepTimeInMs &&
+ !IsComplete() &&
+ !heap()->mark_compact_collector()->marking_deque()->IsEmpty());
+ return remaining_time_in_ms;
+}
void IncrementalMarking::OldSpaceStep(intptr_t allocated) {
- if (IsStopped() && ShouldActivate()) {
- // TODO(hpayer): Let's play safe for now, but compaction should be
- // in principle possible.
- Start(PREVENT_COMPACTION);
+ if (IsStopped() && ShouldActivateEvenWithoutIdleNotification()) {
+ heap()->StartIncrementalMarking(Heap::kNoGCFlags, kNoGCCallbackFlags,
+ "old space step");
} else {
Step(allocated * kFastMarking / kInitialMarkingSpeed, GC_VIA_STACK_GUARD);
}
@@ -809,9 +1060,9 @@
bool speed_up = false;
if ((steps_count_ % kMarkingSpeedAccellerationInterval) == 0) {
- if (FLAG_trace_gc) {
- PrintPID("Speed up marking after %d steps\n",
- static_cast<int>(kMarkingSpeedAccellerationInterval));
+ if (FLAG_trace_incremental_marking) {
+ PrintIsolate(heap()->isolate(), "Speed up marking after %d steps\n",
+ static_cast<int>(kMarkingSpeedAccellerationInterval));
}
speed_up = true;
}
@@ -825,7 +1076,9 @@
if (space_left_is_very_small ||
only_1_nth_of_space_that_was_available_still_left) {
- if (FLAG_trace_gc) PrintPID("Speed up marking because of low space left\n");
+ if (FLAG_trace_incremental_marking)
+ PrintIsolate(heap()->isolate(),
+ "Speed up marking because of low space left\n");
speed_up = true;
}
@@ -835,8 +1088,9 @@
old_generation_space_used_at_start_of_incremental_);
if (size_of_old_space_multiplied_by_n_during_marking) {
speed_up = true;
- if (FLAG_trace_gc) {
- PrintPID("Speed up marking because of heap size increase\n");
+ if (FLAG_trace_incremental_marking) {
+ PrintIsolate(heap()->isolate(),
+ "Speed up marking because of heap size increase\n");
}
}
@@ -848,23 +1102,26 @@
// We try to scan at at least twice the speed that we are allocating.
if (promoted_during_marking > bytes_scanned_ / 2 + scavenge_slack + delay) {
- if (FLAG_trace_gc) {
- PrintPID("Speed up marking because marker was not keeping up\n");
+ if (FLAG_trace_incremental_marking) {
+ PrintIsolate(heap()->isolate(),
+ "Speed up marking because marker was not keeping up\n");
}
speed_up = true;
}
if (speed_up) {
if (state_ != MARKING) {
- if (FLAG_trace_gc) {
- PrintPID("Postponing speeding up marking until marking starts\n");
+ if (FLAG_trace_incremental_marking) {
+ PrintIsolate(heap()->isolate(),
+ "Postponing speeding up marking until marking starts\n");
}
} else {
marking_speed_ += kMarkingSpeedAccelleration;
marking_speed_ = static_cast<int>(
Min(kMaxMarkingSpeed, static_cast<intptr_t>(marking_speed_ * 1.3)));
- if (FLAG_trace_gc) {
- PrintPID("Marking speed increased to %d\n", marking_speed_);
+ if (FLAG_trace_incremental_marking) {
+ PrintIsolate(heap()->isolate(), "Marking speed increased to %d\n",
+ marking_speed_);
}
}
}
@@ -875,8 +1132,9 @@
CompletionAction action,
ForceMarkingAction marking,
ForceCompletionAction completion) {
+ DCHECK(allocated_bytes >= 0);
+
if (heap_->gc_state() != Heap::NOT_IN_GC || !FLAG_incremental_marking ||
- !FLAG_incremental_marking_steps ||
(state_ != SWEEPING && state_ != MARKING)) {
return 0;
}
@@ -901,7 +1159,7 @@
{
HistogramTimerScope incremental_marking_scope(
heap_->isolate()->counters()->gc_incremental_marking());
- double start = base::OS::TimeCurrentMillis();
+ double start = heap_->MonotonicallyIncreasingTimeInMs();
// The marking speed is driven either by the allocation rate or by the rate
// at which we are having to check the color of objects in the write
@@ -927,14 +1185,18 @@
}
if (!heap_->mark_compact_collector()->sweeping_in_progress()) {
bytes_scanned_ = 0;
- StartMarking(PREVENT_COMPACTION);
+ StartMarking();
}
} else if (state_ == MARKING) {
bytes_processed = ProcessMarkingDeque(bytes_to_process);
if (heap_->mark_compact_collector()->marking_deque()->IsEmpty()) {
if (completion == FORCE_COMPLETION ||
IsIdleMarkingDelayCounterLimitReached()) {
- MarkingComplete(action);
+ if (!finalize_marking_completed_) {
+ FinalizeMarking(action);
+ } else {
+ MarkingComplete(action);
+ }
} else {
IncrementIdleMarkingDelayCounter();
}
@@ -947,7 +1209,7 @@
// with marking.
SpeedUp();
- double end = base::OS::TimeCurrentMillis();
+ double end = heap_->MonotonicallyIncreasingTimeInMs();
double duration = (end - start);
// Note that we report zero bytes here when sweeping was in progress or
// when we just started incremental marking. In these cases we did not
@@ -989,5 +1251,5 @@
void IncrementalMarking::ClearIdleMarkingDelayCounter() {
idle_marking_delay_counter_ = 0;
}
-}
-} // namespace v8::internal
+} // namespace internal
+} // namespace v8