src/heap/spaces.cc

// Copyright 2011 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/v8.h"

#include "src/base/bits.h"
#include "src/base/platform/platform.h"
#include "src/full-codegen.h"
#include "src/heap/mark-compact.h"
#include "src/macro-assembler.h"
#include "src/msan.h"

namespace v8 {
namespace internal {


// ----------------------------------------------------------------------------
// HeapObjectIterator

HeapObjectIterator::HeapObjectIterator(PagedSpace* space) {
  // You can't actually iterate over the anchor page.  It is not a real page,
  // just an anchor for the double linked page list.  Initialize as if we have
  // reached the end of the anchor page, then the first iteration will move on
  // to the first page.
  Initialize(space, NULL, NULL, kAllPagesInSpace, NULL);
}


HeapObjectIterator::HeapObjectIterator(PagedSpace* space,
                                       HeapObjectCallback size_func) {
  // You can't actually iterate over the anchor page.  It is not a real page,
  // just an anchor for the double linked page list.  Initialize the current
  // address and end as NULL, then the first iteration will move on
  // to the first page.
  Initialize(space, NULL, NULL, kAllPagesInSpace, size_func);
}


HeapObjectIterator::HeapObjectIterator(Page* page,
                                       HeapObjectCallback size_func) {
  Space* owner = page->owner();
  DCHECK(owner == page->heap()->old_pointer_space() ||
         owner == page->heap()->old_data_space() ||
         owner == page->heap()->map_space() ||
         owner == page->heap()->cell_space() ||
         owner == page->heap()->property_cell_space() ||
         owner == page->heap()->code_space());
  Initialize(reinterpret_cast<PagedSpace*>(owner), page->area_start(),
             page->area_end(), kOnePageOnly, size_func);
  DCHECK(page->WasSwept() || page->SweepingCompleted());
}


void HeapObjectIterator::Initialize(PagedSpace* space, Address cur, Address end,
                                    HeapObjectIterator::PageMode mode,
                                    HeapObjectCallback size_f) {
  space_ = space;
  cur_addr_ = cur;
  cur_end_ = end;
  page_mode_ = mode;
  size_func_ = size_f;
}


// We have hit the end of the page and should advance to the next block of
// objects.  This happens at the end of the page.
bool HeapObjectIterator::AdvanceToNextPage() {
  DCHECK(cur_addr_ == cur_end_);
  if (page_mode_ == kOnePageOnly) return false;
  Page* cur_page;
  if (cur_addr_ == NULL) {
    cur_page = space_->anchor();
  } else {
    cur_page = Page::FromAddress(cur_addr_ - 1);
    DCHECK(cur_addr_ == cur_page->area_end());
  }
  cur_page = cur_page->next_page();
  if (cur_page == space_->anchor()) return false;
  cur_addr_ = cur_page->area_start();
  cur_end_ = cur_page->area_end();
  DCHECK(cur_page->WasSwept() || cur_page->SweepingCompleted());
  return true;
}


// -----------------------------------------------------------------------------
// CodeRange


CodeRange::CodeRange(Isolate* isolate)
    : isolate_(isolate),
      code_range_(NULL),
      free_list_(0),
      allocation_list_(0),
      current_allocation_block_index_(0),
      emergency_block_() {}


bool CodeRange::SetUp(size_t requested) {
  DCHECK(code_range_ == NULL);

  if (requested == 0) {
    // When a target requires the code range feature, we put all code objects
    // in a kMaximalCodeRangeSize range of virtual address space, so that
    // they can call each other with near calls.
    if (kRequiresCodeRange) {
      requested = kMaximalCodeRangeSize;
    } else {
      return true;
    }
  }

  if (requested <= kMinimumCodeRangeSize) {
    requested = kMinimumCodeRangeSize;
  }

  DCHECK(!kRequiresCodeRange || requested <= kMaximalCodeRangeSize);
  code_range_ = new base::VirtualMemory(requested);
  CHECK(code_range_ != NULL);
  if (!code_range_->IsReserved()) {
    delete code_range_;
    code_range_ = NULL;
    return false;
  }

  // We are sure that we have mapped a block of requested addresses.
  DCHECK(code_range_->size() == requested);
  Address base = reinterpret_cast<Address>(code_range_->address());

  // On some platforms, specifically Win64, we need to reserve some pages at
  // the beginning of an executable space.
  if (kReservedCodeRangePages) {
    if (!code_range_->Commit(
            base, kReservedCodeRangePages * base::OS::CommitPageSize(), true)) {
      delete code_range_;
      code_range_ = NULL;
      return false;
    }
    base += kReservedCodeRangePages * base::OS::CommitPageSize();
  }
  Address aligned_base = RoundUp(base, MemoryChunk::kAlignment);
  size_t size = code_range_->size() - (aligned_base - base) -
                kReservedCodeRangePages * base::OS::CommitPageSize();
  allocation_list_.Add(FreeBlock(aligned_base, size));
  current_allocation_block_index_ = 0;

  LOG(isolate_, NewEvent("CodeRange", code_range_->address(), requested));
  ReserveEmergencyBlock();
  return true;
}


int CodeRange::CompareFreeBlockAddress(const FreeBlock* left,
                                       const FreeBlock* right) {
  // The entire point of CodeRange is that the difference between two
  // addresses in the range can be represented as a signed 32-bit int,
  // so the cast is semantically correct.
  return static_cast<int>(left->start - right->start);
}


bool CodeRange::GetNextAllocationBlock(size_t requested) {
  for (current_allocation_block_index_++;
       current_allocation_block_index_ < allocation_list_.length();
       current_allocation_block_index_++) {
    if (requested <= allocation_list_[current_allocation_block_index_].size) {
      return true;  // Found a large enough allocation block.
    }
  }

  // Sort and merge the free blocks on the free list and the allocation list.
  free_list_.AddAll(allocation_list_);
  allocation_list_.Clear();
  free_list_.Sort(&CompareFreeBlockAddress);
  for (int i = 0; i < free_list_.length();) {
    FreeBlock merged = free_list_[i];
    i++;
    // Add adjacent free blocks to the current merged block.
    while (i < free_list_.length() &&
           free_list_[i].start == merged.start + merged.size) {
      merged.size += free_list_[i].size;
      i++;
    }
    if (merged.size > 0) {
      allocation_list_.Add(merged);
    }
  }
  free_list_.Clear();

  for (current_allocation_block_index_ = 0;
       current_allocation_block_index_ < allocation_list_.length();
       current_allocation_block_index_++) {
    if (requested <= allocation_list_[current_allocation_block_index_].size) {
      return true;  // Found a large enough allocation block.
    }
  }
  current_allocation_block_index_ = 0;
  // Code range is full or too fragmented.
  return false;
}


Address CodeRange::AllocateRawMemory(const size_t requested_size,
                                     const size_t commit_size,
                                     size_t* allocated) {
  DCHECK(commit_size <= requested_size);
  FreeBlock current;
  if (!ReserveBlock(requested_size, &current)) {
    *allocated = 0;
    return NULL;
  }
  *allocated = current.size;
  DCHECK(*allocated <= current.size);
  DCHECK(IsAddressAligned(current.start, MemoryChunk::kAlignment));
  if (!isolate_->memory_allocator()->CommitExecutableMemory(
          code_range_, current.start, commit_size, *allocated)) {
    *allocated = 0;
    ReleaseBlock(&current);
    return NULL;
  }
  return current.start;
}


bool CodeRange::CommitRawMemory(Address start, size_t length) {
  return isolate_->memory_allocator()->CommitMemory(start, length, EXECUTABLE);
}


bool CodeRange::UncommitRawMemory(Address start, size_t length) {
  return code_range_->Uncommit(start, length);
}


void CodeRange::FreeRawMemory(Address address, size_t length) {
  DCHECK(IsAddressAligned(address, MemoryChunk::kAlignment));
  free_list_.Add(FreeBlock(address, length));
  code_range_->Uncommit(address, length);
}


void CodeRange::TearDown() {
  delete code_range_;  // Frees all memory in the virtual memory range.
  code_range_ = NULL;
  free_list_.Free();
  allocation_list_.Free();
}


bool CodeRange::ReserveBlock(const size_t requested_size, FreeBlock* block) {
  DCHECK(allocation_list_.length() == 0 ||
         current_allocation_block_index_ < allocation_list_.length());
  if (allocation_list_.length() == 0 ||
      requested_size > allocation_list_[current_allocation_block_index_].size) {
    // Find an allocation block large enough.
    if (!GetNextAllocationBlock(requested_size)) return false;
  }
  // Commit the requested memory at the start of the current allocation block.
  size_t aligned_requested = RoundUp(requested_size, MemoryChunk::kAlignment);
  *block = allocation_list_[current_allocation_block_index_];
  // Don't leave a small free block, useless for a large object or chunk.
  if (aligned_requested < (block->size - Page::kPageSize)) {
    block->size = aligned_requested;
  }
  DCHECK(IsAddressAligned(block->start, MemoryChunk::kAlignment));
  allocation_list_[current_allocation_block_index_].start += block->size;
  allocation_list_[current_allocation_block_index_].size -= block->size;
  return true;
}


void CodeRange::ReleaseBlock(const FreeBlock* block) { free_list_.Add(*block); }


void CodeRange::ReserveEmergencyBlock() {
  const size_t requested_size = MemoryAllocator::CodePageAreaSize();
  if (emergency_block_.size == 0) {
    ReserveBlock(requested_size, &emergency_block_);
  } else {
    DCHECK(emergency_block_.size >= requested_size);
  }
}


void CodeRange::ReleaseEmergencyBlock() {
  if (emergency_block_.size != 0) {
    ReleaseBlock(&emergency_block_);
    emergency_block_.size = 0;
  }
}


// -----------------------------------------------------------------------------
// MemoryAllocator
//

MemoryAllocator::MemoryAllocator(Isolate* isolate)
    : isolate_(isolate),
      capacity_(0),
      capacity_executable_(0),
      size_(0),
      size_executable_(0),
      lowest_ever_allocated_(reinterpret_cast<void*>(-1)),
      highest_ever_allocated_(reinterpret_cast<void*>(0)) {}


bool MemoryAllocator::SetUp(intptr_t capacity, intptr_t capacity_executable) {
  capacity_ = RoundUp(capacity, Page::kPageSize);
  capacity_executable_ = RoundUp(capacity_executable, Page::kPageSize);
  DCHECK_GE(capacity_, capacity_executable_);

  size_ = 0;
  size_executable_ = 0;

  return true;
}


void MemoryAllocator::TearDown() {
  // Check that spaces were torn down before MemoryAllocator.
  DCHECK(size_ == 0);
  // TODO(gc) this will be true again when we fix FreeMemory.
  // DCHECK(size_executable_ == 0);
  capacity_ = 0;
  capacity_executable_ = 0;
}


bool MemoryAllocator::CommitMemory(Address base, size_t size,
                                   Executability executable) {
  if (!base::VirtualMemory::CommitRegion(base, size,
                                         executable == EXECUTABLE)) {
    return false;
  }
  UpdateAllocatedSpaceLimits(base, base + size);
  return true;
}


void MemoryAllocator::FreeMemory(base::VirtualMemory* reservation,
                                 Executability executable) {
  // TODO(gc) make code_range part of memory allocator?
  DCHECK(reservation->IsReserved());
  size_t size = reservation->size();
  DCHECK(size_ >= size);
  size_ -= size;

  isolate_->counters()->memory_allocated()->Decrement(static_cast<int>(size));

  if (executable == EXECUTABLE) {
    DCHECK(size_executable_ >= size);
    size_executable_ -= size;
  }
  // Code which is part of the code-range does not have its own VirtualMemory.
  DCHECK(isolate_->code_range() == NULL ||
         !isolate_->code_range()->contains(
             static_cast<Address>(reservation->address())));
  DCHECK(executable == NOT_EXECUTABLE || isolate_->code_range() == NULL ||
         !isolate_->code_range()->valid());
  reservation->Release();
}


void MemoryAllocator::FreeMemory(Address base, size_t size,
                                 Executability executable) {
  // TODO(gc) make code_range part of memory allocator?
  DCHECK(size_ >= size);
  size_ -= size;

  isolate_->counters()->memory_allocated()->Decrement(static_cast<int>(size));

  if (executable == EXECUTABLE) {
    DCHECK(size_executable_ >= size);
    size_executable_ -= size;
  }
  if (isolate_->code_range() != NULL &&
      isolate_->code_range()->contains(static_cast<Address>(base))) {
    DCHECK(executable == EXECUTABLE);
    isolate_->code_range()->FreeRawMemory(base, size);
  } else {
    DCHECK(executable == NOT_EXECUTABLE || isolate_->code_range() == NULL ||
           !isolate_->code_range()->valid());
    bool result = base::VirtualMemory::ReleaseRegion(base, size);
    USE(result);
    DCHECK(result);
  }
}


Address MemoryAllocator::ReserveAlignedMemory(size_t size, size_t alignment,
                                              base::VirtualMemory* controller) {
  base::VirtualMemory reservation(size, alignment);

  if (!reservation.IsReserved()) return NULL;
  size_ += reservation.size();
  Address base =
      RoundUp(static_cast<Address>(reservation.address()), alignment);
  controller->TakeControl(&reservation);
  return base;
}


Address MemoryAllocator::AllocateAlignedMemory(
    size_t reserve_size, size_t commit_size, size_t alignment,
    Executability executable, base::VirtualMemory* controller) {
  DCHECK(commit_size <= reserve_size);
  base::VirtualMemory reservation;
  Address base = ReserveAlignedMemory(reserve_size, alignment, &reservation);
  if (base == NULL) return NULL;

  if (executable == EXECUTABLE) {
    if (!CommitExecutableMemory(&reservation, base, commit_size,
                                reserve_size)) {
      base = NULL;
    }
  } else {
    if (reservation.Commit(base, commit_size, false)) {
      UpdateAllocatedSpaceLimits(base, base + commit_size);
    } else {
      base = NULL;
    }
  }

  if (base == NULL) {
    // Failed to commit the body. Release the mapping and any partially
    // commited regions inside it.
    reservation.Release();
    return NULL;
  }

  controller->TakeControl(&reservation);
  return base;
}


void Page::InitializeAsAnchor(PagedSpace* owner) {
  set_owner(owner);
  set_prev_page(this);
  set_next_page(this);
}


NewSpacePage* NewSpacePage::Initialize(Heap* heap, Address start,
                                       SemiSpace* semi_space) {
  Address area_start = start + NewSpacePage::kObjectStartOffset;
  Address area_end = start + Page::kPageSize;

  MemoryChunk* chunk =
      MemoryChunk::Initialize(heap, start, Page::kPageSize, area_start,
                              area_end, NOT_EXECUTABLE, semi_space);
  chunk->set_next_chunk(NULL);
  chunk->set_prev_chunk(NULL);
  chunk->initialize_scan_on_scavenge(true);
  bool in_to_space = (semi_space->id() != kFromSpace);
  chunk->SetFlag(in_to_space ? MemoryChunk::IN_TO_SPACE
                             : MemoryChunk::IN_FROM_SPACE);
  DCHECK(!chunk->IsFlagSet(in_to_space ? MemoryChunk::IN_FROM_SPACE
                                       : MemoryChunk::IN_TO_SPACE));
  NewSpacePage* page = static_cast<NewSpacePage*>(chunk);
  heap->incremental_marking()->SetNewSpacePageFlags(page);
  return page;
}


void NewSpacePage::InitializeAsAnchor(SemiSpace* semi_space) {
  set_owner(semi_space);
  set_next_chunk(this);
  set_prev_chunk(this);
  // Flags marks this invalid page as not being in new-space.
  // All real new-space pages will be in new-space.
  SetFlags(0, ~0);
}


MemoryChunk* MemoryChunk::Initialize(Heap* heap, Address base, size_t size,
                                     Address area_start, Address area_end,
                                     Executability executable, Space* owner) {
  MemoryChunk* chunk = FromAddress(base);

  DCHECK(base == chunk->address());

  chunk->heap_ = heap;
  chunk->size_ = size;
  chunk->area_start_ = area_start;
  chunk->area_end_ = area_end;
  chunk->flags_ = 0;
  chunk->set_owner(owner);
  chunk->InitializeReservedMemory();
  chunk->slots_buffer_ = NULL;
  chunk->skip_list_ = NULL;
  chunk->write_barrier_counter_ = kWriteBarrierCounterGranularity;
  chunk->progress_bar_ = 0;
  chunk->high_water_mark_ = static_cast<int>(area_start - base);
  chunk->set_parallel_sweeping(SWEEPING_DONE);
  chunk->available_in_small_free_list_ = 0;
  chunk->available_in_medium_free_list_ = 0;
  chunk->available_in_large_free_list_ = 0;
  chunk->available_in_huge_free_list_ = 0;
  chunk->non_available_small_blocks_ = 0;
  chunk->ResetLiveBytes();
  Bitmap::Clear(chunk);
  chunk->initialize_scan_on_scavenge(false);
  chunk->SetFlag(WAS_SWEPT);

  DCHECK(OFFSET_OF(MemoryChunk, flags_) == kFlagsOffset);
  DCHECK(OFFSET_OF(MemoryChunk, live_byte_count_) == kLiveBytesOffset);

  if (executable == EXECUTABLE) {
    chunk->SetFlag(IS_EXECUTABLE);
  }

  if (owner == heap->old_data_space()) {
    chunk->SetFlag(CONTAINS_ONLY_DATA);
  }

  return chunk;
}


// Commit MemoryChunk area to the requested size.
bool MemoryChunk::CommitArea(size_t requested) {
  size_t guard_size =
      IsFlagSet(IS_EXECUTABLE) ? MemoryAllocator::CodePageGuardSize() : 0;
  size_t header_size = area_start() - address() - guard_size;
  size_t commit_size =
      RoundUp(header_size + requested, base::OS::CommitPageSize());
  size_t committed_size = RoundUp(header_size + (area_end() - area_start()),
                                  base::OS::CommitPageSize());

  if (commit_size > committed_size) {
    // Commit size should be less or equal than the reserved size.
    DCHECK(commit_size <= size() - 2 * guard_size);
    // Append the committed area.
    Address start = address() + committed_size + guard_size;
    size_t length = commit_size - committed_size;
    if (reservation_.IsReserved()) {
      Executability executable =
          IsFlagSet(IS_EXECUTABLE) ? EXECUTABLE : NOT_EXECUTABLE;
      if (!heap()->isolate()->memory_allocator()->CommitMemory(start, length,
                                                               executable)) {
        return false;
      }
    } else {
      CodeRange* code_range = heap_->isolate()->code_range();
      DCHECK(code_range != NULL && code_range->valid() &&
             IsFlagSet(IS_EXECUTABLE));
      if (!code_range->CommitRawMemory(start, length)) return false;
    }

    if (Heap::ShouldZapGarbage()) {
      heap_->isolate()->memory_allocator()->ZapBlock(start, length);
    }
  } else if (commit_size < committed_size) {
    DCHECK(commit_size > 0);
    // Shrink the committed area.
    size_t length = committed_size - commit_size;
    Address start = address() + committed_size + guard_size - length;
    if (reservation_.IsReserved()) {
      if (!reservation_.Uncommit(start, length)) return false;
    } else {
      CodeRange* code_range = heap_->isolate()->code_range();
      DCHECK(code_range != NULL && code_range->valid() &&
             IsFlagSet(IS_EXECUTABLE));
      if (!code_range->UncommitRawMemory(start, length)) return false;
    }
  }

  area_end_ = area_start_ + requested;
  return true;
}


void MemoryChunk::InsertAfter(MemoryChunk* other) {
  MemoryChunk* other_next = other->next_chunk();

  set_next_chunk(other_next);
  set_prev_chunk(other);
  other_next->set_prev_chunk(this);
  other->set_next_chunk(this);
}


void MemoryChunk::Unlink() {
  MemoryChunk* next_element = next_chunk();
  MemoryChunk* prev_element = prev_chunk();
  next_element->set_prev_chunk(prev_element);
  prev_element->set_next_chunk(next_element);
  set_prev_chunk(NULL);
  set_next_chunk(NULL);
}


MemoryChunk* MemoryAllocator::AllocateChunk(intptr_t reserve_area_size,
                                            intptr_t commit_area_size,
                                            Executability executable,
                                            Space* owner) {
  DCHECK(commit_area_size <= reserve_area_size);

  size_t chunk_size;
  Heap* heap = isolate_->heap();
  Address base = NULL;
  base::VirtualMemory reservation;
  Address area_start = NULL;
  Address area_end = NULL;

  //
  // MemoryChunk layout:
  //
  //             Executable
  // +----------------------------+<- base aligned with MemoryChunk::kAlignment
  // |           Header           |
  // +----------------------------+<- base + CodePageGuardStartOffset
  // |           Guard            |
  // +----------------------------+<- area_start_
  // |           Area             |
  // +----------------------------+<- area_end_ (area_start + commit_area_size)
  // |   Committed but not used   |
  // +----------------------------+<- aligned at OS page boundary
  // | Reserved but not committed |
  // +----------------------------+<- aligned at OS page boundary
  // |           Guard            |
  // +----------------------------+<- base + chunk_size
  //
  //           Non-executable
  // +----------------------------+<- base aligned with MemoryChunk::kAlignment
  // |          Header            |
  // +----------------------------+<- area_start_ (base + kObjectStartOffset)
  // |           Area             |
  // +----------------------------+<- area_end_ (area_start + commit_area_size)
  // |  Committed but not used    |
  // +----------------------------+<- aligned at OS page boundary
  // | Reserved but not committed |
  // +----------------------------+<- base + chunk_size
  //

  if (executable == EXECUTABLE) {
    chunk_size = RoundUp(CodePageAreaStartOffset() + reserve_area_size,
                         base::OS::CommitPageSize()) +
                 CodePageGuardSize();

    // Check executable memory limit.
    if (size_executable_ + chunk_size > capacity_executable_) {
      LOG(isolate_, StringEvent("MemoryAllocator::AllocateRawMemory",
                                "V8 Executable Allocation capacity exceeded"));
      return NULL;
    }

    // Size of header (not executable) plus area (executable).
    size_t commit_size = RoundUp(CodePageGuardStartOffset() + commit_area_size,
                                 base::OS::CommitPageSize());
    // Allocate executable memory either from code range or from the
    // OS.
    if (isolate_->code_range() != NULL && isolate_->code_range()->valid()) {
      base = isolate_->code_range()->AllocateRawMemory(chunk_size, commit_size,
                                                       &chunk_size);
      DCHECK(
          IsAligned(reinterpret_cast<intptr_t>(base), MemoryChunk::kAlignment));
      if (base == NULL) return NULL;
      size_ += chunk_size;
      // Update executable memory size.
      size_executable_ += chunk_size;
    } else {
      base = AllocateAlignedMemory(chunk_size, commit_size,
                                   MemoryChunk::kAlignment, executable,
                                   &reservation);
      if (base == NULL) return NULL;
      // Update executable memory size.
      size_executable_ += reservation.size();
    }

    if (Heap::ShouldZapGarbage()) {
      ZapBlock(base, CodePageGuardStartOffset());
      ZapBlock(base + CodePageAreaStartOffset(), commit_area_size);
    }

    area_start = base + CodePageAreaStartOffset();
    area_end = area_start + commit_area_size;
  } else {
    chunk_size = RoundUp(MemoryChunk::kObjectStartOffset + reserve_area_size,
                         base::OS::CommitPageSize());
    size_t commit_size =
        RoundUp(MemoryChunk::kObjectStartOffset + commit_area_size,
                base::OS::CommitPageSize());
    base =
        AllocateAlignedMemory(chunk_size, commit_size, MemoryChunk::kAlignment,
                              executable, &reservation);

    if (base == NULL) return NULL;

    if (Heap::ShouldZapGarbage()) {
      ZapBlock(base, Page::kObjectStartOffset + commit_area_size);
    }

    area_start = base + Page::kObjectStartOffset;
    area_end = area_start + commit_area_size;
  }

  // Use chunk_size for statistics and callbacks because we assume that they
  // treat reserved but not-yet committed memory regions of chunks as allocated.
  isolate_->counters()->memory_allocated()->Increment(
      static_cast<int>(chunk_size));

  LOG(isolate_, NewEvent("MemoryChunk", base, chunk_size));
  if (owner != NULL) {
    ObjectSpace space = static_cast<ObjectSpace>(1 << owner->identity());
    PerformAllocationCallback(space, kAllocationActionAllocate, chunk_size);
  }

  MemoryChunk* result = MemoryChunk::Initialize(
      heap, base, chunk_size, area_start, area_end, executable, owner);
  result->set_reserved_memory(&reservation);
  return result;
}


void Page::ResetFreeListStatistics() {
  non_available_small_blocks_ = 0;
  available_in_small_free_list_ = 0;
  available_in_medium_free_list_ = 0;
  available_in_large_free_list_ = 0;
  available_in_huge_free_list_ = 0;
}


Page* MemoryAllocator::AllocatePage(intptr_t size, PagedSpace* owner,
                                    Executability executable) {
  MemoryChunk* chunk = AllocateChunk(size, size, executable, owner);

  if (chunk == NULL) return NULL;

  return Page::Initialize(isolate_->heap(), chunk, executable, owner);
}


LargePage* MemoryAllocator::AllocateLargePage(intptr_t object_size,
                                              Space* owner,
                                              Executability executable) {
  MemoryChunk* chunk =
      AllocateChunk(object_size, object_size, executable, owner);
  if (chunk == NULL) return NULL;
  return LargePage::Initialize(isolate_->heap(), chunk);
}


void MemoryAllocator::Free(MemoryChunk* chunk) {
  LOG(isolate_, DeleteEvent("MemoryChunk", chunk));
  if (chunk->owner() != NULL) {
    ObjectSpace space =
        static_cast<ObjectSpace>(1 << chunk->owner()->identity());
    PerformAllocationCallback(space, kAllocationActionFree, chunk->size());
  }

  isolate_->heap()->RememberUnmappedPage(reinterpret_cast<Address>(chunk),
                                         chunk->IsEvacuationCandidate());

  delete chunk->slots_buffer();
  delete chunk->skip_list();

  base::VirtualMemory* reservation = chunk->reserved_memory();
  if (reservation->IsReserved()) {
    FreeMemory(reservation, chunk->executable());
  } else {
    FreeMemory(chunk->address(), chunk->size(), chunk->executable());
  }
}


bool MemoryAllocator::CommitBlock(Address start, size_t size,
                                  Executability executable) {
  if (!CommitMemory(start, size, executable)) return false;

  if (Heap::ShouldZapGarbage()) {
    ZapBlock(start, size);
  }

  isolate_->counters()->memory_allocated()->Increment(static_cast<int>(size));
  return true;
}


bool MemoryAllocator::UncommitBlock(Address start, size_t size) {
  if (!base::VirtualMemory::UncommitRegion(start, size)) return false;
  isolate_->counters()->memory_allocated()->Decrement(static_cast<int>(size));
  return true;
}


void MemoryAllocator::ZapBlock(Address start, size_t size) {
  for (size_t s = 0; s + kPointerSize <= size; s += kPointerSize) {
    Memory::Address_at(start + s) = kZapValue;
  }
}


void MemoryAllocator::PerformAllocationCallback(ObjectSpace space,
                                                AllocationAction action,
                                                size_t size) {
  for (int i = 0; i < memory_allocation_callbacks_.length(); ++i) {
    MemoryAllocationCallbackRegistration registration =
        memory_allocation_callbacks_[i];
    if ((registration.space & space) == space &&
        (registration.action & action) == action)
      registration.callback(space, action, static_cast<int>(size));
  }
}


bool MemoryAllocator::MemoryAllocationCallbackRegistered(
    MemoryAllocationCallback callback) {
  for (int i = 0; i < memory_allocation_callbacks_.length(); ++i) {
    if (memory_allocation_callbacks_[i].callback == callback) return true;
  }
  return false;
}


void MemoryAllocator::AddMemoryAllocationCallback(
    MemoryAllocationCallback callback, ObjectSpace space,
    AllocationAction action) {
  DCHECK(callback != NULL);
  MemoryAllocationCallbackRegistration registration(callback, space, action);
  DCHECK(!MemoryAllocator::MemoryAllocationCallbackRegistered(callback));
  return memory_allocation_callbacks_.Add(registration);
}


void MemoryAllocator::RemoveMemoryAllocationCallback(
    MemoryAllocationCallback callback) {
  DCHECK(callback != NULL);
  for (int i = 0; i < memory_allocation_callbacks_.length(); ++i) {
    if (memory_allocation_callbacks_[i].callback == callback) {
      memory_allocation_callbacks_.Remove(i);
      return;
    }
  }
  UNREACHABLE();
}


#ifdef DEBUG
void MemoryAllocator::ReportStatistics() {
  float pct = static_cast<float>(capacity_ - size_) / capacity_;
  PrintF("  capacity: %" V8_PTR_PREFIX
         "d"
         ", used: %" V8_PTR_PREFIX
         "d"
         ", available: %%%d\n\n",
         capacity_, size_, static_cast<int>(pct * 100));
}
#endif


int MemoryAllocator::CodePageGuardStartOffset() {
  // We are guarding code pages: the first OS page after the header
  // will be protected as non-writable.
  return RoundUp(Page::kObjectStartOffset, base::OS::CommitPageSize());
}


int MemoryAllocator::CodePageGuardSize() {
  return static_cast<int>(base::OS::CommitPageSize());
}


int MemoryAllocator::CodePageAreaStartOffset() {
  // We are guarding code pages: the first OS page after the header
  // will be protected as non-writable.
  return CodePageGuardStartOffset() + CodePageGuardSize();
}


int MemoryAllocator::CodePageAreaEndOffset() {
  // We are guarding code pages: the last OS page will be protected as
  // non-writable.
  return Page::kPageSize - static_cast<int>(base::OS::CommitPageSize());
}


bool MemoryAllocator::CommitExecutableMemory(base::VirtualMemory* vm,
                                             Address start, size_t commit_size,
                                             size_t reserved_size) {
  // Commit page header (not executable).
  if (!vm->Commit(start, CodePageGuardStartOffset(), false)) {
    return false;
  }

  // Create guard page after the header.
  if (!vm->Guard(start + CodePageGuardStartOffset())) {
    return false;
  }

  // Commit page body (executable).
  if (!vm->Commit(start + CodePageAreaStartOffset(),
                  commit_size - CodePageGuardStartOffset(), true)) {
    return false;
  }

  // Create guard page before the end.
  if (!vm->Guard(start + reserved_size - CodePageGuardSize())) {
    return false;
  }

  UpdateAllocatedSpaceLimits(start, start + CodePageAreaStartOffset() +
                                        commit_size -
                                        CodePageGuardStartOffset());
  return true;
}


// -----------------------------------------------------------------------------
// MemoryChunk implementation

void MemoryChunk::IncrementLiveBytesFromMutator(Address address, int by) {
  MemoryChunk* chunk = MemoryChunk::FromAddress(address);
  if (!chunk->InNewSpace() && !static_cast<Page*>(chunk)->WasSwept()) {
    static_cast<PagedSpace*>(chunk->owner())->IncrementUnsweptFreeBytes(-by);
  }
  chunk->IncrementLiveBytes(by);
}


// -----------------------------------------------------------------------------
// PagedSpace implementation

STATIC_ASSERT(static_cast<ObjectSpace>(1 << AllocationSpace::NEW_SPACE) ==
              ObjectSpace::kObjectSpaceNewSpace);
STATIC_ASSERT(static_cast<ObjectSpace>(1
                                       << AllocationSpace::OLD_POINTER_SPACE) ==
              ObjectSpace::kObjectSpaceOldPointerSpace);
STATIC_ASSERT(static_cast<ObjectSpace>(1 << AllocationSpace::OLD_DATA_SPACE) ==
              ObjectSpace::kObjectSpaceOldDataSpace);
STATIC_ASSERT(static_cast<ObjectSpace>(1 << AllocationSpace::CODE_SPACE) ==
              ObjectSpace::kObjectSpaceCodeSpace);
STATIC_ASSERT(static_cast<ObjectSpace>(1 << AllocationSpace::CELL_SPACE) ==
              ObjectSpace::kObjectSpaceCellSpace);
STATIC_ASSERT(
    static_cast<ObjectSpace>(1 << AllocationSpace::PROPERTY_CELL_SPACE) ==
    ObjectSpace::kObjectSpacePropertyCellSpace);
STATIC_ASSERT(static_cast<ObjectSpace>(1 << AllocationSpace::MAP_SPACE) ==
              ObjectSpace::kObjectSpaceMapSpace);


PagedSpace::PagedSpace(Heap* heap, intptr_t max_capacity, AllocationSpace space,
                       Executability executable)
    : Space(heap, space, executable),
      free_list_(this),
      unswept_free_bytes_(0),
      end_of_unswept_pages_(NULL),
      emergency_memory_(NULL) {
  area_size_ = MemoryAllocator::PageAreaSize(space);
  max_capacity_ =
      (RoundDown(max_capacity, Page::kPageSize) / Page::kPageSize) * AreaSize();
  accounting_stats_.Clear();

  allocation_info_.set_top(NULL);
  allocation_info_.set_limit(NULL);

  anchor_.InitializeAsAnchor(this);
}


bool PagedSpace::SetUp() { return true; }


bool PagedSpace::HasBeenSetUp() { return true; }


void PagedSpace::TearDown() {
  PageIterator iterator(this);
  while (iterator.has_next()) {
    heap()->isolate()->memory_allocator()->Free(iterator.next());
  }
  anchor_.set_next_page(&anchor_);
  anchor_.set_prev_page(&anchor_);
  accounting_stats_.Clear();
}


size_t PagedSpace::CommittedPhysicalMemory() {
  if (!base::VirtualMemory::HasLazyCommits()) return CommittedMemory();
  MemoryChunk::UpdateHighWaterMark(allocation_info_.top());
  size_t size = 0;
  PageIterator it(this);
  while (it.has_next()) {
    size += it.next()->CommittedPhysicalMemory();
  }
  return size;
}


Object* PagedSpace::FindObject(Address addr) {
  // Note: this function can only be called on iterable spaces.
  DCHECK(!heap()->mark_compact_collector()->in_use());

  if (!Contains(addr)) return Smi::FromInt(0);  // Signaling not found.

  Page* p = Page::FromAddress(addr);
  HeapObjectIterator it(p, NULL);
  for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
    Address cur = obj->address();
    Address next = cur + obj->Size();
    if ((cur <= addr) && (addr < next)) return obj;
  }

  UNREACHABLE();
  return Smi::FromInt(0);
}


bool PagedSpace::CanExpand() {
  DCHECK(max_capacity_ % AreaSize() == 0);

  if (Capacity() == max_capacity_) return false;

  DCHECK(Capacity() < max_capacity_);

  // Are we going to exceed capacity for this space?
  if ((Capacity() + Page::kPageSize) > max_capacity_) return false;

  return true;
}


bool PagedSpace::Expand() {
  if (!CanExpand()) return false;

  intptr_t size = AreaSize();

  if (anchor_.next_page() == &anchor_) {
    size = SizeOfFirstPage();
  }

  Page* p = heap()->isolate()->memory_allocator()->AllocatePage(size, this,
                                                                executable());
  if (p == NULL) return false;

  DCHECK(Capacity() <= max_capacity_);

  p->InsertAfter(anchor_.prev_page());

  return true;
}


intptr_t PagedSpace::SizeOfFirstPage() {
  // If using an ool constant pool then transfer the constant pool allowance
  // from the code space to the old pointer space.
  static const int constant_pool_delta = FLAG_enable_ool_constant_pool ? 48 : 0;
  int size = 0;
  switch (identity()) {
    case OLD_POINTER_SPACE:
      size = (128 + constant_pool_delta) * kPointerSize * KB;
      break;
    case OLD_DATA_SPACE:
      size = 192 * KB;
      break;
    case MAP_SPACE:
      size = 16 * kPointerSize * KB;
      break;
    case CELL_SPACE:
      size = 16 * kPointerSize * KB;
      break;
    case PROPERTY_CELL_SPACE:
      size = 8 * kPointerSize * KB;
      break;
    case CODE_SPACE: {
      CodeRange* code_range = heap()->isolate()->code_range();
      if (code_range != NULL && code_range->valid()) {
        // When code range exists, code pages are allocated in a special way
        // (from the reserved code range). That part of the code is not yet
        // upgraded to handle small pages.
        size = AreaSize();
      } else {
        size = RoundUp((480 - constant_pool_delta) * KB *
                           FullCodeGenerator::kBootCodeSizeMultiplier / 100,
                       kPointerSize);
      }
      break;
    }
    default:
      UNREACHABLE();
  }
  return Min(size, AreaSize());
}


int PagedSpace::CountTotalPages() {
  PageIterator it(this);
  int count = 0;
  while (it.has_next()) {
    it.next();
    count++;
  }
  return count;
}


void PagedSpace::ObtainFreeListStatistics(Page* page, SizeStats* sizes) {
  sizes->huge_size_ = page->available_in_huge_free_list();
  sizes->small_size_ = page->available_in_small_free_list();
  sizes->medium_size_ = page->available_in_medium_free_list();
  sizes->large_size_ = page->available_in_large_free_list();
}


void PagedSpace::ResetFreeListStatistics() {
  PageIterator page_iterator(this);
  while (page_iterator.has_next()) {
    Page* page = page_iterator.next();
    page->ResetFreeListStatistics();
  }
}


void PagedSpace::IncreaseCapacity(int size) {
  accounting_stats_.ExpandSpace(size);
}


void PagedSpace::ReleasePage(Page* page) {
  DCHECK(page->LiveBytes() == 0);
  DCHECK(AreaSize() == page->area_size());

  if (page->WasSwept()) {
    intptr_t size = free_list_.EvictFreeListItems(page);
    accounting_stats_.AllocateBytes(size);
    DCHECK_EQ(AreaSize(), static_cast<int>(size));
  } else {
    DecreaseUnsweptFreeBytes(page);
  }

  if (page->IsFlagSet(MemoryChunk::SCAN_ON_SCAVENGE)) {
    heap()->decrement_scan_on_scavenge_pages();
    page->ClearFlag(MemoryChunk::SCAN_ON_SCAVENGE);
  }

  DCHECK(!free_list_.ContainsPageFreeListItems(page));

  if (Page::FromAllocationTop(allocation_info_.top()) == page) {
    allocation_info_.set_top(NULL);
    allocation_info_.set_limit(NULL);
  }

  // If page is still in a list, unlink it from that list.
  if (page->next_chunk() != NULL) {
    DCHECK(page->prev_chunk() != NULL);
    page->Unlink();
  }

  if (page->IsFlagSet(MemoryChunk::CONTAINS_ONLY_DATA)) {
    heap()->isolate()->memory_allocator()->Free(page);
  } else {
    heap()->QueueMemoryChunkForFree(page);
  }

  DCHECK(Capacity() > 0);
  accounting_stats_.ShrinkSpace(AreaSize());
}


void PagedSpace::CreateEmergencyMemory() {
  if (identity() == CODE_SPACE) {
    // Make the emergency block available to the allocator.
    CodeRange* code_range = heap()->isolate()->code_range();
    if (code_range != NULL && code_range->valid()) {
      code_range->ReleaseEmergencyBlock();
    }
    DCHECK(MemoryAllocator::CodePageAreaSize() == AreaSize());
  }
  emergency_memory_ = heap()->isolate()->memory_allocator()->AllocateChunk(
      AreaSize(), AreaSize(), executable(), this);
}


void PagedSpace::FreeEmergencyMemory() {
  Page* page = static_cast<Page*>(emergency_memory_);
  DCHECK(page->LiveBytes() == 0);
  DCHECK(AreaSize() == page->area_size());
  DCHECK(!free_list_.ContainsPageFreeListItems(page));
  heap()->isolate()->memory_allocator()->Free(page);
  emergency_memory_ = NULL;
}


void PagedSpace::UseEmergencyMemory() {
  Page* page = Page::Initialize(heap(), emergency_memory_, executable(), this);
  page->InsertAfter(anchor_.prev_page());
  emergency_memory_ = NULL;
}


#ifdef DEBUG
void PagedSpace::Print() {}
#endif

#ifdef VERIFY_HEAP
void PagedSpace::Verify(ObjectVisitor* visitor) {
  bool allocation_pointer_found_in_space =
      (allocation_info_.top() == allocation_info_.limit());
  PageIterator page_iterator(this);
  while (page_iterator.has_next()) {
    Page* page = page_iterator.next();
    CHECK(page->owner() == this);
    if (page == Page::FromAllocationTop(allocation_info_.top())) {
      allocation_pointer_found_in_space = true;
    }
    CHECK(page->WasSwept());
    HeapObjectIterator it(page, NULL);
    Address end_of_previous_object = page->area_start();
    Address top = page->area_end();
    int black_size = 0;
    for (HeapObject* object = it.Next(); object != NULL; object = it.Next()) {
      CHECK(end_of_previous_object <= object->address());

      // The first word should be a map, and we expect all map pointers to
      // be in map space.
      Map* map = object->map();
      CHECK(map->IsMap());
      CHECK(heap()->map_space()->Contains(map));

      // Perform space-specific object verification.
      VerifyObject(object);

      // The object itself should look OK.
      object->ObjectVerify();

      // All the interior pointers should be contained in the heap.
      int size = object->Size();
      object->IterateBody(map->instance_type(), size, visitor);
      if (Marking::IsBlack(Marking::MarkBitFrom(object))) {
        black_size += size;
      }

      CHECK(object->address() + size <= top);
      end_of_previous_object = object->address() + size;
    }
    CHECK_LE(black_size, page->LiveBytes());
  }
  CHECK(allocation_pointer_found_in_space);
}
#endif  // VERIFY_HEAP

// -----------------------------------------------------------------------------
// NewSpace implementation


bool NewSpace::SetUp(int reserved_semispace_capacity,
                     int maximum_semispace_capacity) {
  // Set up new space based on the preallocated memory block defined by
  // start and size. The provided space is divided into two semi-spaces.
  // To support fast containment testing in the new space, the size of
  // this chunk must be a power of two and it must be aligned to its size.
  int initial_semispace_capacity = heap()->InitialSemiSpaceSize();

  int target_semispace_capacity = heap()->TargetSemiSpaceSize();

  size_t size = 2 * reserved_semispace_capacity;
  Address base = heap()->isolate()->memory_allocator()->ReserveAlignedMemory(
      size, size, &reservation_);
  if (base == NULL) return false;

  chunk_base_ = base;
  chunk_size_ = static_cast<uintptr_t>(size);
  LOG(heap()->isolate(), NewEvent("InitialChunk", chunk_base_, chunk_size_));

  DCHECK(initial_semispace_capacity <= maximum_semispace_capacity);
  DCHECK(base::bits::IsPowerOfTwo32(maximum_semispace_capacity));

  // Allocate and set up the histogram arrays if necessary.
  allocated_histogram_ = NewArray<HistogramInfo>(LAST_TYPE + 1);
  promoted_histogram_ = NewArray<HistogramInfo>(LAST_TYPE + 1);

#define SET_NAME(name)                        \
  allocated_histogram_[name].set_name(#name); \
  promoted_histogram_[name].set_name(#name);
  INSTANCE_TYPE_LIST(SET_NAME)
#undef SET_NAME

  DCHECK(reserved_semispace_capacity == heap()->ReservedSemiSpaceSize());
  DCHECK(static_cast<intptr_t>(chunk_size_) >=
         2 * heap()->ReservedSemiSpaceSize());
  DCHECK(IsAddressAligned(chunk_base_, 2 * reserved_semispace_capacity, 0));

  to_space_.SetUp(chunk_base_, initial_semispace_capacity,
                  target_semispace_capacity, maximum_semispace_capacity);
  from_space_.SetUp(chunk_base_ + reserved_semispace_capacity,
                    initial_semispace_capacity, target_semispace_capacity,
                    maximum_semispace_capacity);
  if (!to_space_.Commit()) {
    return false;
  }
  DCHECK(!from_space_.is_committed());  // No need to use memory yet.

  start_ = chunk_base_;
  address_mask_ = ~(2 * reserved_semispace_capacity - 1);
  object_mask_ = address_mask_ | kHeapObjectTagMask;
  object_expected_ = reinterpret_cast<uintptr_t>(start_) | kHeapObjectTag;

  ResetAllocationInfo();

  return true;
}


void NewSpace::TearDown() {
  if (allocated_histogram_) {
    DeleteArray(allocated_histogram_);
    allocated_histogram_ = NULL;
  }
  if (promoted_histogram_) {
    DeleteArray(promoted_histogram_);
    promoted_histogram_ = NULL;
  }

  start_ = NULL;
  allocation_info_.set_top(NULL);
  allocation_info_.set_limit(NULL);

  to_space_.TearDown();
  from_space_.TearDown();

  LOG(heap()->isolate(), DeleteEvent("InitialChunk", chunk_base_));

  DCHECK(reservation_.IsReserved());
  heap()->isolate()->memory_allocator()->FreeMemory(&reservation_,
                                                    NOT_EXECUTABLE);
  chunk_base_ = NULL;
  chunk_size_ = 0;
}


void NewSpace::Flip() { SemiSpace::Swap(&from_space_, &to_space_); }


void NewSpace::Grow() {
  // Double the semispace size but only up to maximum capacity.
  DCHECK(TotalCapacity() < MaximumCapacity());
  int new_capacity =
      Min(MaximumCapacity(),
          FLAG_semi_space_growth_factor * static_cast<int>(TotalCapacity()));
  if (to_space_.GrowTo(new_capacity)) {
    // Only grow from space if we managed to grow to-space.
    if (!from_space_.GrowTo(new_capacity)) {
      // If we managed to grow to-space but couldn't grow from-space,
      // attempt to shrink to-space.
      if (!to_space_.ShrinkTo(from_space_.TotalCapacity())) {
        // We are in an inconsistent state because we could not
        // commit/uncommit memory from new space.
        CHECK(false);
      }
    }
  }
  DCHECK_SEMISPACE_ALLOCATION_INFO(allocation_info_, to_space_);
}


bool NewSpace::GrowOnePage() {
  if (TotalCapacity() == MaximumCapacity()) return false;
  int new_capacity = static_cast<int>(TotalCapacity()) + Page::kPageSize;
  if (to_space_.GrowTo(new_capacity)) {
    // Only grow from space if we managed to grow to-space and the from space
    // is actually committed.
    if (from_space_.is_committed()) {
      if (!from_space_.GrowTo(new_capacity)) {
        // If we managed to grow to-space but couldn't grow from-space,
        // attempt to shrink to-space.
        if (!to_space_.ShrinkTo(from_space_.TotalCapacity())) {
          // We are in an inconsistent state because we could not
          // commit/uncommit memory from new space.
          CHECK(false);
        }
        return false;
      }
    } else {
      if (!from_space_.SetTotalCapacity(new_capacity)) {
        // Can't really happen, but better safe than sorry.
        CHECK(false);
      }
    }
    DCHECK_SEMISPACE_ALLOCATION_INFO(allocation_info_, to_space_);
    return true;
  }
  return false;
}


void NewSpace::Shrink() {
  int new_capacity = Max(InitialTotalCapacity(), 2 * SizeAsInt());
  int rounded_new_capacity = RoundUp(new_capacity, Page::kPageSize);
  if (rounded_new_capacity < TotalCapacity() &&
      to_space_.ShrinkTo(rounded_new_capacity)) {
    // Only shrink from-space if we managed to shrink to-space.
    from_space_.Reset();
    if (!from_space_.ShrinkTo(rounded_new_capacity)) {
      // If we managed to shrink to-space but couldn't shrink from
      // space, attempt to grow to-space again.
      if (!to_space_.GrowTo(from_space_.TotalCapacity())) {
        // We are in an inconsistent state because we could not
        // commit/uncommit memory from new space.
        CHECK(false);
      }
    }
  }
  DCHECK_SEMISPACE_ALLOCATION_INFO(allocation_info_, to_space_);
}


void NewSpace::UpdateAllocationInfo() {
  MemoryChunk::UpdateHighWaterMark(allocation_info_.top());
  allocation_info_.set_top(to_space_.page_low());
  allocation_info_.set_limit(to_space_.page_high());
  UpdateInlineAllocationLimit(0);
  DCHECK_SEMISPACE_ALLOCATION_INFO(allocation_info_, to_space_);
}


void NewSpace::ResetAllocationInfo() {
  to_space_.Reset();
  UpdateAllocationInfo();
  pages_used_ = 0;
  // Clear all mark-bits in the to-space.
  NewSpacePageIterator it(&to_space_);
  while (it.has_next()) {
    Bitmap::Clear(it.next());
  }
}


void NewSpace::UpdateInlineAllocationLimit(int size_in_bytes) {
  if (heap()->inline_allocation_disabled()) {
    // Lowest limit when linear allocation was disabled.
    Address high = to_space_.page_high();
    Address new_top = allocation_info_.top() + size_in_bytes;
    allocation_info_.set_limit(Min(new_top, high));
  } else if (inline_allocation_limit_step() == 0) {
    // Normal limit is the end of the current page.
    allocation_info_.set_limit(to_space_.page_high());
  } else {
    // Lower limit during incremental marking.
    Address high = to_space_.page_high();
    Address new_top = allocation_info_.top() + size_in_bytes;
    Address new_limit = new_top + inline_allocation_limit_step_;
    allocation_info_.set_limit(Min(new_limit, high));
  }
  DCHECK_SEMISPACE_ALLOCATION_INFO(allocation_info_, to_space_);
}


bool NewSpace::AddFreshPage() {
  Address top = allocation_info_.top();
  if (NewSpacePage::IsAtStart(top)) {
    // The current page is already empty. Don't try to make another.

    // We should only get here if someone asks to allocate more
    // than what can be stored in a single page.
    // TODO(gc): Change the limit on new-space allocation to prevent this
    // from happening (all such allocations should go directly to LOSpace).
    return false;
  }
  if (!to_space_.AdvancePage()) {
    // Check if we reached the target capacity yet. If not, try to commit a page
    // and continue.
    if ((to_space_.TotalCapacity() < to_space_.TargetCapacity()) &&
        GrowOnePage()) {
      if (!to_space_.AdvancePage()) {
        // It doesn't make sense that we managed to commit a page, but can't use
        // it.
        CHECK(false);
      }
    } else {
      // Failed to get a new page in to-space.
      return false;
    }
  }

  // Clear remainder of current page.
  Address limit = NewSpacePage::FromLimit(top)->area_end();
  if (heap()->gc_state() == Heap::SCAVENGE) {
    heap()->promotion_queue()->SetNewLimit(limit);
  }

  int remaining_in_page = static_cast<int>(limit - top);
  heap()->CreateFillerObjectAt(top, remaining_in_page);
  pages_used_++;
  UpdateAllocationInfo();

  return true;
}


AllocationResult NewSpace::SlowAllocateRaw(int size_in_bytes) {
  Address old_top = allocation_info_.top();
  Address high = to_space_.page_high();
  if (allocation_info_.limit() < high) {
    // Either the limit has been lowered because linear allocation was disabled
    // or because incremental marking wants to get a chance to do a step. Set
    // the new limit accordingly.
    Address new_top = old_top + size_in_bytes;
    int bytes_allocated = static_cast<int>(new_top - top_on_previous_step_);
    heap()->incremental_marking()->Step(bytes_allocated,
                                        IncrementalMarking::GC_VIA_STACK_GUARD);
    UpdateInlineAllocationLimit(size_in_bytes);
    top_on_previous_step_ = new_top;
    return AllocateRaw(size_in_bytes);
  } else if (AddFreshPage()) {
    // Switched to new page. Try allocating again.
    int bytes_allocated = static_cast<int>(old_top - top_on_previous_step_);
    heap()->incremental_marking()->Step(bytes_allocated,
                                        IncrementalMarking::GC_VIA_STACK_GUARD);
    top_on_previous_step_ = to_space_.page_low();
    return AllocateRaw(size_in_bytes);
  } else {
    return AllocationResult::Retry();
  }
}


#ifdef VERIFY_HEAP
// We do not use the SemiSpaceIterator because verification doesn't assume
// that it works (it depends on the invariants we are checking).
void NewSpace::Verify() {
  // The allocation pointer should be in the space or at the very end.
  DCHECK_SEMISPACE_ALLOCATION_INFO(allocation_info_, to_space_);

  // There should be objects packed in from the low address up to the
  // allocation pointer.
  Address current = to_space_.first_page()->area_start();
  CHECK_EQ(current, to_space_.space_start());

  while (current != top()) {
    if (!NewSpacePage::IsAtEnd(current)) {
      // The allocation pointer should not be in the middle of an object.
      CHECK(!NewSpacePage::FromLimit(current)->ContainsLimit(top()) ||
            current < top());

      HeapObject* object = HeapObject::FromAddress(current);

      // The first word should be a map, and we expect all map pointers to
      // be in map space.
      Map* map = object->map();
      CHECK(map->IsMap());
      CHECK(heap()->map_space()->Contains(map));

      // The object should not be code or a map.
      CHECK(!object->IsMap());
      CHECK(!object->IsCode());

      // The object itself should look OK.
      object->ObjectVerify();

      // All the interior pointers should be contained in the heap.
      VerifyPointersVisitor visitor;
      int size = object->Size();
      object->IterateBody(map->instance_type(), size, &visitor);

      current += size;
    } else {
      // At end of page, switch to next page.
      NewSpacePage* page = NewSpacePage::FromLimit(current)->next_page();
      // Next page should be valid.
      CHECK(!page->is_anchor());
      current = page->area_start();
    }
  }

  // Check semi-spaces.
  CHECK_EQ(from_space_.id(), kFromSpace);
  CHECK_EQ(to_space_.id(), kToSpace);
  from_space_.Verify();
  to_space_.Verify();
}
#endif

// -----------------------------------------------------------------------------
// SemiSpace implementation

void SemiSpace::SetUp(Address start, int initial_capacity, int target_capacity,
                      int maximum_capacity) {
  // Creates a space in the young generation. The constructor does not
  // allocate memory from the OS.  A SemiSpace is given a contiguous chunk of
  // memory of size 'capacity' when set up, and does not grow or shrink
  // otherwise.  In the mark-compact collector, the memory region of the from
  // space is used as the marking stack. It requires contiguous memory
  // addresses.
  DCHECK(maximum_capacity >= Page::kPageSize);
  DCHECK(initial_capacity <= target_capacity);
  DCHECK(target_capacity <= maximum_capacity);
  initial_total_capacity_ = RoundDown(initial_capacity, Page::kPageSize);
  total_capacity_ = initial_capacity;
  target_capacity_ = RoundDown(target_capacity, Page::kPageSize);
  maximum_total_capacity_ = RoundDown(maximum_capacity, Page::kPageSize);
  maximum_committed_ = 0;
  committed_ = false;
  start_ = start;
  address_mask_ = ~(maximum_capacity - 1);
  object_mask_ = address_mask_ | kHeapObjectTagMask;
  object_expected_ = reinterpret_cast<uintptr_t>(start) | kHeapObjectTag;
  age_mark_ = start_;
}


void SemiSpace::TearDown() {
  start_ = NULL;
  total_capacity_ = 0;
}


bool SemiSpace::Commit() {
  DCHECK(!is_committed());
  int pages = total_capacity_ / Page::kPageSize;
  if (!heap()->isolate()->memory_allocator()->CommitBlock(
          start_, total_capacity_, executable())) {
    return false;
  }

  NewSpacePage* current = anchor();
  for (int i = 0; i < pages; i++) {
    NewSpacePage* new_page =
        NewSpacePage::Initialize(heap(), start_ + i * Page::kPageSize, this);
    new_page->InsertAfter(current);
    current = new_page;
  }

  SetCapacity(total_capacity_);
  committed_ = true;
  Reset();
  return true;
}


bool SemiSpace::Uncommit() {
  DCHECK(is_committed());
  Address start = start_ + maximum_total_capacity_ - total_capacity_;
  if (!heap()->isolate()->memory_allocator()->UncommitBlock(start,
                                                            total_capacity_)) {
    return false;
  }
  anchor()->set_next_page(anchor());
  anchor()->set_prev_page(anchor());

  committed_ = false;
  return true;
}


size_t SemiSpace::CommittedPhysicalMemory() {
  if (!is_committed()) return 0;
  size_t size = 0;
  NewSpacePageIterator it(this);
  while (it.has_next()) {
    size += it.next()->CommittedPhysicalMemory();
  }
  return size;
}


bool SemiSpace::GrowTo(int new_capacity) {
  if (!is_committed()) {
    if (!Commit()) return false;
  }
  DCHECK((new_capacity & Page::kPageAlignmentMask) == 0);
  DCHECK(new_capacity <= maximum_total_capacity_);
  DCHECK(new_capacity > total_capacity_);
  int pages_before = total_capacity_ / Page::kPageSize;
  int pages_after = new_capacity / Page::kPageSize;

  size_t delta = new_capacity - total_capacity_;

  DCHECK(IsAligned(delta, base::OS::AllocateAlignment()));
  if (!heap()->isolate()->memory_allocator()->CommitBlock(
          start_ + total_capacity_, delta, executable())) {
    return false;
  }
  SetCapacity(new_capacity);
  NewSpacePage* last_page = anchor()->prev_page();
  DCHECK(last_page != anchor());
  for (int i = pages_before; i < pages_after; i++) {
    Address page_address = start_ + i * Page::kPageSize;
    NewSpacePage* new_page =
        NewSpacePage::Initialize(heap(), page_address, this);
    new_page->InsertAfter(last_page);
    Bitmap::Clear(new_page);
    // Duplicate the flags that was set on the old page.
    new_page->SetFlags(last_page->GetFlags(),
                       NewSpacePage::kCopyOnFlipFlagsMask);
    last_page = new_page;
  }
  return true;
}


bool SemiSpace::ShrinkTo(int new_capacity) {
  DCHECK((new_capacity & Page::kPageAlignmentMask) == 0);
  DCHECK(new_capacity >= initial_total_capacity_);
  DCHECK(new_capacity < total_capacity_);
  if (is_committed()) {
    size_t delta = total_capacity_ - new_capacity;
    DCHECK(IsAligned(delta, base::OS::AllocateAlignment()));

    MemoryAllocator* allocator = heap()->isolate()->memory_allocator();
    if (!allocator->UncommitBlock(start_ + new_capacity, delta)) {
      return false;
    }

    int pages_after = new_capacity / Page::kPageSize;
    NewSpacePage* new_last_page =
        NewSpacePage::FromAddress(start_ + (pages_after - 1) * Page::kPageSize);
    new_last_page->set_next_page(anchor());
    anchor()->set_prev_page(new_last_page);
    DCHECK((current_page_ >= first_page()) && (current_page_ <= new_last_page));
  }

  SetCapacity(new_capacity);

  return true;
}


bool SemiSpace::SetTotalCapacity(int new_capacity) {
  CHECK(!is_committed());
  if (new_capacity >= initial_total_capacity_ &&
      new_capacity <= maximum_total_capacity_) {
    total_capacity_ = new_capacity;
    return true;
  }
  return false;
}


void SemiSpace::FlipPages(intptr_t flags, intptr_t mask) {
  anchor_.set_owner(this);
  // Fixup back-pointers to anchor. Address of anchor changes
  // when we swap.
  anchor_.prev_page()->set_next_page(&anchor_);
  anchor_.next_page()->set_prev_page(&anchor_);

  bool becomes_to_space = (id_ == kFromSpace);
  id_ = becomes_to_space ? kToSpace : kFromSpace;
  NewSpacePage* page = anchor_.next_page();
  while (page != &anchor_) {
    page->set_owner(this);
    page->SetFlags(flags, mask);
    if (becomes_to_space) {
      page->ClearFlag(MemoryChunk::IN_FROM_SPACE);
      page->SetFlag(MemoryChunk::IN_TO_SPACE);
      page->ClearFlag(MemoryChunk::NEW_SPACE_BELOW_AGE_MARK);
      page->ResetLiveBytes();
    } else {
      page->SetFlag(MemoryChunk::IN_FROM_SPACE);
      page->ClearFlag(MemoryChunk::IN_TO_SPACE);
    }
    DCHECK(page->IsFlagSet(MemoryChunk::SCAN_ON_SCAVENGE));
    DCHECK(page->IsFlagSet(MemoryChunk::IN_TO_SPACE) ||
           page->IsFlagSet(MemoryChunk::IN_FROM_SPACE));
    page = page->next_page();
  }
}


void SemiSpace::Reset() {
  DCHECK(anchor_.next_page() != &anchor_);
  current_page_ = anchor_.next_page();
}


void SemiSpace::Swap(SemiSpace* from, SemiSpace* to) {
  // We won't be swapping semispaces without data in them.
  DCHECK(from->anchor_.next_page() != &from->anchor_);
  DCHECK(to->anchor_.next_page() != &to->anchor_);

  // Swap bits.
  SemiSpace tmp = *from;
  *from = *to;
  *to = tmp;

  // Fixup back-pointers to the page list anchor now that its address
  // has changed.
  // Swap to/from-space bits on pages.
  // Copy GC flags from old active space (from-space) to new (to-space).
  intptr_t flags = from->current_page()->GetFlags();
  to->FlipPages(flags, NewSpacePage::kCopyOnFlipFlagsMask);

  from->FlipPages(0, 0);
}


void SemiSpace::SetCapacity(int new_capacity) {
  total_capacity_ = new_capacity;
  if (total_capacity_ > maximum_committed_) {
    maximum_committed_ = total_capacity_;
  }
}


void SemiSpace::set_age_mark(Address mark) {
  DCHECK(NewSpacePage::FromLimit(mark)->semi_space() == this);
  age_mark_ = mark;
  // Mark all pages up to the one containing mark.
  NewSpacePageIterator it(space_start(), mark);
  while (it.has_next()) {
    it.next()->SetFlag(MemoryChunk::NEW_SPACE_BELOW_AGE_MARK);
  }
}


#ifdef DEBUG
void SemiSpace::Print() {}
#endif

#ifdef VERIFY_HEAP
void SemiSpace::Verify() {
  bool is_from_space = (id_ == kFromSpace);
  NewSpacePage* page = anchor_.next_page();
  CHECK(anchor_.semi_space() == this);
  while (page != &anchor_) {
    CHECK(page->semi_space() == this);
    CHECK(page->InNewSpace());
    CHECK(page->IsFlagSet(is_from_space ? MemoryChunk::IN_FROM_SPACE
                                        : MemoryChunk::IN_TO_SPACE));
    CHECK(!page->IsFlagSet(is_from_space ? MemoryChunk::IN_TO_SPACE
                                         : MemoryChunk::IN_FROM_SPACE));
    CHECK(page->IsFlagSet(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING));
    if (!is_from_space) {
      // The pointers-from-here-are-interesting flag isn't updated dynamically
      // on from-space pages, so it might be out of sync with the marking state.
      if (page->heap()->incremental_marking()->IsMarking()) {
        CHECK(page->IsFlagSet(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING));
      } else {
        CHECK(
            !page->IsFlagSet(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING));
      }
      // TODO(gc): Check that the live_bytes_count_ field matches the
      // black marking on the page (if we make it match in new-space).
    }
    CHECK(page->IsFlagSet(MemoryChunk::SCAN_ON_SCAVENGE));
    CHECK(page->prev_page()->next_page() == page);
    page = page->next_page();
  }
}
#endif

#ifdef DEBUG
void SemiSpace::AssertValidRange(Address start, Address end) {
  // Addresses belong to same semi-space
  NewSpacePage* page = NewSpacePage::FromLimit(start);
  NewSpacePage* end_page = NewSpacePage::FromLimit(end);
  SemiSpace* space = page->semi_space();
  CHECK_EQ(space, end_page->semi_space());
  // Start address is before end address, either on same page,
  // or end address is on a later page in the linked list of
  // semi-space pages.
  if (page == end_page) {
    CHECK(start <= end);
  } else {
    while (page != end_page) {
      page = page->next_page();
      CHECK_NE(page, space->anchor());
    }
  }
}
#endif


// -----------------------------------------------------------------------------
// SemiSpaceIterator implementation.
SemiSpaceIterator::SemiSpaceIterator(NewSpace* space) {
  Initialize(space->bottom(), space->top(), NULL);
}


SemiSpaceIterator::SemiSpaceIterator(NewSpace* space,
                                     HeapObjectCallback size_func) {
  Initialize(space->bottom(), space->top(), size_func);
}


SemiSpaceIterator::SemiSpaceIterator(NewSpace* space, Address start) {
  Initialize(start, space->top(), NULL);
}


SemiSpaceIterator::SemiSpaceIterator(Address from, Address to) {
  Initialize(from, to, NULL);
}


void SemiSpaceIterator::Initialize(Address start, Address end,
                                   HeapObjectCallback size_func) {
  SemiSpace::AssertValidRange(start, end);
  current_ = start;
  limit_ = end;
  size_func_ = size_func;
}


#ifdef DEBUG
// heap_histograms is shared, always clear it before using it.
static void ClearHistograms(Isolate* isolate) {
// We reset the name each time, though it hasn't changed.
#define DEF_TYPE_NAME(name) isolate->heap_histograms()[name].set_name(#name);
  INSTANCE_TYPE_LIST(DEF_TYPE_NAME)
#undef DEF_TYPE_NAME

#define CLEAR_HISTOGRAM(name) isolate->heap_histograms()[name].clear();
  INSTANCE_TYPE_LIST(CLEAR_HISTOGRAM)
#undef CLEAR_HISTOGRAM

  isolate->js_spill_information()->Clear();
}


static void ClearCodeKindStatistics(int* code_kind_statistics) {
  for (int i = 0; i < Code::NUMBER_OF_KINDS; i++) {
    code_kind_statistics[i] = 0;
  }
}


static void ReportCodeKindStatistics(int* code_kind_statistics) {
  PrintF("\n   Code kind histograms: \n");
  for (int i = 0; i < Code::NUMBER_OF_KINDS; i++) {
    if (code_kind_statistics[i] > 0) {
      PrintF("     %-20s: %10d bytes\n",
             Code::Kind2String(static_cast<Code::Kind>(i)),
             code_kind_statistics[i]);
    }
  }
  PrintF("\n");
}


static int CollectHistogramInfo(HeapObject* obj) {
  Isolate* isolate = obj->GetIsolate();
  InstanceType type = obj->map()->instance_type();
  DCHECK(0 <= type && type <= LAST_TYPE);
  DCHECK(isolate->heap_histograms()[type].name() != NULL);
  isolate->heap_histograms()[type].increment_number(1);
  isolate->heap_histograms()[type].increment_bytes(obj->Size());

  if (FLAG_collect_heap_spill_statistics && obj->IsJSObject()) {
    JSObject::cast(obj)
        ->IncrementSpillStatistics(isolate->js_spill_information());
  }

  return obj->Size();
}


static void ReportHistogram(Isolate* isolate, bool print_spill) {
  PrintF("\n  Object Histogram:\n");
  for (int i = 0; i <= LAST_TYPE; i++) {
    if (isolate->heap_histograms()[i].number() > 0) {
      PrintF("    %-34s%10d (%10d bytes)\n",
             isolate->heap_histograms()[i].name(),
             isolate->heap_histograms()[i].number(),
             isolate->heap_histograms()[i].bytes());
    }
  }
  PrintF("\n");

  // Summarize string types.
  int string_number = 0;
  int string_bytes = 0;
#define INCREMENT(type, size, name, camel_name)               \
  string_number += isolate->heap_histograms()[type].number(); \
  string_bytes += isolate->heap_histograms()[type].bytes();
  STRING_TYPE_LIST(INCREMENT)
#undef INCREMENT
  if (string_number > 0) {
    PrintF("    %-34s%10d (%10d bytes)\n\n", "STRING_TYPE", string_number,
           string_bytes);
  }

  if (FLAG_collect_heap_spill_statistics && print_spill) {
    isolate->js_spill_information()->Print();
  }
}
#endif  // DEBUG


// Support for statistics gathering for --heap-stats and --log-gc.
void NewSpace::ClearHistograms() {
  for (int i = 0; i <= LAST_TYPE; i++) {
    allocated_histogram_[i].clear();
    promoted_histogram_[i].clear();
  }
}


// Because the copying collector does not touch garbage objects, we iterate
// the new space before a collection to get a histogram of allocated objects.
// This only happens when --log-gc flag is set.
void NewSpace::CollectStatistics() {
  ClearHistograms();
  SemiSpaceIterator it(this);
  for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next())
    RecordAllocation(obj);
}


static void DoReportStatistics(Isolate* isolate, HistogramInfo* info,
                               const char* description) {
  LOG(isolate, HeapSampleBeginEvent("NewSpace", description));
  // Lump all the string types together.
  int string_number = 0;
  int string_bytes = 0;
#define INCREMENT(type, size, name, camel_name) \
  string_number += info[type].number();         \
  string_bytes += info[type].bytes();
  STRING_TYPE_LIST(INCREMENT)
#undef INCREMENT
  if (string_number > 0) {
    LOG(isolate,
        HeapSampleItemEvent("STRING_TYPE", string_number, string_bytes));
  }

  // Then do the other types.
  for (int i = FIRST_NONSTRING_TYPE; i <= LAST_TYPE; ++i) {
    if (info[i].number() > 0) {
      LOG(isolate, HeapSampleItemEvent(info[i].name(), info[i].number(),
                                       info[i].bytes()));
    }
  }
  LOG(isolate, HeapSampleEndEvent("NewSpace", description));
}


void NewSpace::ReportStatistics() {
#ifdef DEBUG
  if (FLAG_heap_stats) {
    float pct = static_cast<float>(Available()) / TotalCapacity();
    PrintF("  capacity: %" V8_PTR_PREFIX
           "d"
           ", available: %" V8_PTR_PREFIX "d, %%%d\n",
           TotalCapacity(), Available(), static_cast<int>(pct * 100));
    PrintF("\n  Object Histogram:\n");
    for (int i = 0; i <= LAST_TYPE; i++) {
      if (allocated_histogram_[i].number() > 0) {
        PrintF("    %-34s%10d (%10d bytes)\n", allocated_histogram_[i].name(),
               allocated_histogram_[i].number(),
               allocated_histogram_[i].bytes());
      }
    }
    PrintF("\n");
  }
#endif  // DEBUG

  if (FLAG_log_gc) {
    Isolate* isolate = heap()->isolate();
    DoReportStatistics(isolate, allocated_histogram_, "allocated");
    DoReportStatistics(isolate, promoted_histogram_, "promoted");
  }
}


void NewSpace::RecordAllocation(HeapObject* obj) {
  InstanceType type = obj->map()->instance_type();
  DCHECK(0 <= type && type <= LAST_TYPE);
  allocated_histogram_[type].increment_number(1);
  allocated_histogram_[type].increment_bytes(obj->Size());
}


void NewSpace::RecordPromotion(HeapObject* obj) {
  InstanceType type = obj->map()->instance_type();
  DCHECK(0 <= type && type <= LAST_TYPE);
  promoted_histogram_[type].increment_number(1);
  promoted_histogram_[type].increment_bytes(obj->Size());
}


size_t NewSpace::CommittedPhysicalMemory() {
  if (!base::VirtualMemory::HasLazyCommits()) return CommittedMemory();
  MemoryChunk::UpdateHighWaterMark(allocation_info_.top());
  size_t size = to_space_.CommittedPhysicalMemory();
  if (from_space_.is_committed()) {
    size += from_space_.CommittedPhysicalMemory();
  }
  return size;
}


// -----------------------------------------------------------------------------
// Free lists for old object spaces implementation

void FreeListNode::set_size(Heap* heap, int size_in_bytes) {
  DCHECK(size_in_bytes > 0);
  DCHECK(IsAligned(size_in_bytes, kPointerSize));

  // We write a map and possibly size information to the block.  If the block
  // is big enough to be a FreeSpace with at least one extra word (the next
  // pointer), we set its map to be the free space map and its size to an
  // appropriate array length for the desired size from HeapObject::Size().
  // If the block is too small (eg, one or two words), to hold both a size
  // field and a next pointer, we give it a filler map that gives it the
  // correct size.
  if (size_in_bytes > FreeSpace::kHeaderSize) {
    // Can't use FreeSpace::cast because it fails during deserialization.
    // We have to set the size first with a release store before we store
    // the map because a concurrent store buffer scan on scavenge must not
    // observe a map with an invalid size.
    FreeSpace* this_as_free_space = reinterpret_cast<FreeSpace*>(this);
    this_as_free_space->nobarrier_set_size(size_in_bytes);
    synchronized_set_map_no_write_barrier(heap->raw_unchecked_free_space_map());
  } else if (size_in_bytes == kPointerSize) {
    set_map_no_write_barrier(heap->raw_unchecked_one_pointer_filler_map());
  } else if (size_in_bytes == 2 * kPointerSize) {
    set_map_no_write_barrier(heap->raw_unchecked_two_pointer_filler_map());
  } else {
    UNREACHABLE();
  }
  // We would like to DCHECK(Size() == size_in_bytes) but this would fail during
  // deserialization because the free space map is not done yet.
}


FreeListNode* FreeListNode::next() {
  DCHECK(IsFreeListNode(this));
  if (map() == GetHeap()->raw_unchecked_free_space_map()) {
    DCHECK(map() == NULL || Size() >= kNextOffset + kPointerSize);
    return reinterpret_cast<FreeListNode*>(
        Memory::Address_at(address() + kNextOffset));
  } else {
    return reinterpret_cast<FreeListNode*>(
        Memory::Address_at(address() + kPointerSize));
  }
}


FreeListNode** FreeListNode::next_address() {
  DCHECK(IsFreeListNode(this));
  if (map() == GetHeap()->raw_unchecked_free_space_map()) {
    DCHECK(Size() >= kNextOffset + kPointerSize);
    return reinterpret_cast<FreeListNode**>(address() + kNextOffset);
  } else {
    return reinterpret_cast<FreeListNode**>(address() + kPointerSize);
  }
}


void FreeListNode::set_next(FreeListNode* next) {
  DCHECK(IsFreeListNode(this));
  // While we are booting the VM the free space map will actually be null.  So
  // we have to make sure that we don't try to use it for anything at that
  // stage.
  if (map() == GetHeap()->raw_unchecked_free_space_map()) {
    DCHECK(map() == NULL || Size() >= kNextOffset + kPointerSize);
    base::NoBarrier_Store(
        reinterpret_cast<base::AtomicWord*>(address() + kNextOffset),
        reinterpret_cast<base::AtomicWord>(next));
  } else {
    base::NoBarrier_Store(
        reinterpret_cast<base::AtomicWord*>(address() + kPointerSize),
        reinterpret_cast<base::AtomicWord>(next));
  }
}


intptr_t FreeListCategory::Concatenate(FreeListCategory* category) {
  intptr_t free_bytes = 0;
  if (category->top() != NULL) {
    // This is safe (not going to deadlock) since Concatenate operations
    // are never performed on the same free lists at the same time in
    // reverse order.
    base::LockGuard<base::Mutex> target_lock_guard(mutex());
    base::LockGuard<base::Mutex> source_lock_guard(category->mutex());
    DCHECK(category->end_ != NULL);
    free_bytes = category->available();
    if (end_ == NULL) {
      end_ = category->end();
    } else {
      category->end()->set_next(top());
    }
    set_top(category->top());
    base::NoBarrier_Store(&top_, category->top_);
    available_ += category->available();
    category->Reset();
  }
  return free_bytes;
}


void FreeListCategory::Reset() {
  set_top(NULL);
  set_end(NULL);
  set_available(0);
}


intptr_t FreeListCategory::EvictFreeListItemsInList(Page* p) {
  int sum = 0;
  FreeListNode* t = top();
  FreeListNode** n = &t;
  while (*n != NULL) {
    if (Page::FromAddress((*n)->address()) == p) {
      FreeSpace* free_space = reinterpret_cast<FreeSpace*>(*n);
      sum += free_space->Size();
      *n = (*n)->next();
    } else {
      n = (*n)->next_address();
    }
  }
  set_top(t);
  if (top() == NULL) {
    set_end(NULL);
  }
  available_ -= sum;
  return sum;
}


bool FreeListCategory::ContainsPageFreeListItemsInList(Page* p) {
  FreeListNode* node = top();
  while (node != NULL) {
    if (Page::FromAddress(node->address()) == p) return true;
    node = node->next();
  }
  return false;
}


FreeListNode* FreeListCategory::PickNodeFromList(int* node_size) {
  FreeListNode* node = top();

  if (node == NULL) return NULL;

  while (node != NULL &&
         Page::FromAddress(node->address())->IsEvacuationCandidate()) {
    available_ -= reinterpret_cast<FreeSpace*>(node)->Size();
    node = node->next();
  }

  if (node != NULL) {
    set_top(node->next());
    *node_size = reinterpret_cast<FreeSpace*>(node)->Size();
    available_ -= *node_size;
  } else {
    set_top(NULL);
  }

  if (top() == NULL) {
    set_end(NULL);
  }

  return node;
}


FreeListNode* FreeListCategory::PickNodeFromList(int size_in_bytes,
                                                 int* node_size) {
  FreeListNode* node = PickNodeFromList(node_size);
  if (node != NULL && *node_size < size_in_bytes) {
    Free(node, *node_size);
    *node_size = 0;
    return NULL;
  }
  return node;
}


void FreeListCategory::Free(FreeListNode* node, int size_in_bytes) {
  node->set_next(top());
  set_top(node);
  if (end_ == NULL) {
    end_ = node;
  }
  available_ += size_in_bytes;
}


void FreeListCategory::RepairFreeList(Heap* heap) {
  FreeListNode* n = top();
  while (n != NULL) {
    Map** map_location = reinterpret_cast<Map**>(n->address());
    if (*map_location == NULL) {
      *map_location = heap->free_space_map();
    } else {
      DCHECK(*map_location == heap->free_space_map());
    }
    n = n->next();
  }
}


FreeList::FreeList(PagedSpace* owner) : owner_(owner), heap_(owner->heap()) {
  Reset();
}


intptr_t FreeList::Concatenate(FreeList* free_list) {
  intptr_t free_bytes = 0;
  free_bytes += small_list_.Concatenate(free_list->small_list());
  free_bytes += medium_list_.Concatenate(free_list->medium_list());
  free_bytes += large_list_.Concatenate(free_list->large_list());
  free_bytes += huge_list_.Concatenate(free_list->huge_list());
  return free_bytes;
}


void FreeList::Reset() {
  small_list_.Reset();
  medium_list_.Reset();
  large_list_.Reset();
  huge_list_.Reset();
}


int FreeList::Free(Address start, int size_in_bytes) {
  if (size_in_bytes == 0) return 0;

  FreeListNode* node = FreeListNode::FromAddress(start);
  node->set_size(heap_, size_in_bytes);
  Page* page = Page::FromAddress(start);

  // Early return to drop too-small blocks on the floor.
  if (size_in_bytes < kSmallListMin) {
    page->add_non_available_small_blocks(size_in_bytes);
    return size_in_bytes;
  }

  // Insert other blocks at the head of a free list of the appropriate
  // magnitude.
  if (size_in_bytes <= kSmallListMax) {
    small_list_.Free(node, size_in_bytes);
    page->add_available_in_small_free_list(size_in_bytes);
  } else if (size_in_bytes <= kMediumListMax) {
    medium_list_.Free(node, size_in_bytes);
    page->add_available_in_medium_free_list(size_in_bytes);
  } else if (size_in_bytes <= kLargeListMax) {
    large_list_.Free(node, size_in_bytes);
    page->add_available_in_large_free_list(size_in_bytes);
  } else {
    huge_list_.Free(node, size_in_bytes);
    page->add_available_in_huge_free_list(size_in_bytes);
  }

  DCHECK(IsVeryLong() || available() == SumFreeLists());
  return 0;
}


FreeListNode* FreeList::FindNodeFor(int size_in_bytes, int* node_size) {
  FreeListNode* node = NULL;
  Page* page = NULL;

  if (size_in_bytes <= kSmallAllocationMax) {
    node = small_list_.PickNodeFromList(node_size);
    if (node != NULL) {
      DCHECK(size_in_bytes <= *node_size);
      page = Page::FromAddress(node->address());
      page->add_available_in_small_free_list(-(*node_size));
      DCHECK(IsVeryLong() || available() == SumFreeLists());
      return node;
    }
  }

  if (size_in_bytes <= kMediumAllocationMax) {
    node = medium_list_.PickNodeFromList(node_size);
    if (node != NULL) {
      DCHECK(size_in_bytes <= *node_size);
      page = Page::FromAddress(node->address());
      page->add_available_in_medium_free_list(-(*node_size));
      DCHECK(IsVeryLong() || available() == SumFreeLists());
      return node;
    }
  }

  if (size_in_bytes <= kLargeAllocationMax) {
    node = large_list_.PickNodeFromList(node_size);
    if (node != NULL) {
      DCHECK(size_in_bytes <= *node_size);
      page = Page::FromAddress(node->address());
      page->add_available_in_large_free_list(-(*node_size));
      DCHECK(IsVeryLong() || available() == SumFreeLists());
      return node;
    }
  }

  int huge_list_available = huge_list_.available();
  FreeListNode* top_node = huge_list_.top();
  for (FreeListNode** cur = &top_node; *cur != NULL;
       cur = (*cur)->next_address()) {
    FreeListNode* cur_node = *cur;
    while (cur_node != NULL &&
           Page::FromAddress(cur_node->address())->IsEvacuationCandidate()) {
      int size = reinterpret_cast<FreeSpace*>(cur_node)->Size();
      huge_list_available -= size;
      page = Page::FromAddress(cur_node->address());
      page->add_available_in_huge_free_list(-size);
      cur_node = cur_node->next();
    }

    *cur = cur_node;
    if (cur_node == NULL) {
      huge_list_.set_end(NULL);
      break;
    }

    DCHECK((*cur)->map() == heap_->raw_unchecked_free_space_map());
    FreeSpace* cur_as_free_space = reinterpret_cast<FreeSpace*>(*cur);
    int size = cur_as_free_space->Size();
    if (size >= size_in_bytes) {
      // Large enough node found.  Unlink it from the list.
      node = *cur;
      *cur = node->next();
      *node_size = size;
      huge_list_available -= size;
      page = Page::FromAddress(node->address());
      page->add_available_in_huge_free_list(-size);
      break;
    }
  }

  huge_list_.set_top(top_node);
  if (huge_list_.top() == NULL) {
    huge_list_.set_end(NULL);
  }
  huge_list_.set_available(huge_list_available);

  if (node != NULL) {
    DCHECK(IsVeryLong() || available() == SumFreeLists());
    return node;
  }

  if (size_in_bytes <= kSmallListMax) {
    node = small_list_.PickNodeFromList(size_in_bytes, node_size);
    if (node != NULL) {
      DCHECK(size_in_bytes <= *node_size);
      page = Page::FromAddress(node->address());
      page->add_available_in_small_free_list(-(*node_size));
    }
  } else if (size_in_bytes <= kMediumListMax) {
    node = medium_list_.PickNodeFromList(size_in_bytes, node_size);
    if (node != NULL) {
      DCHECK(size_in_bytes <= *node_size);
      page = Page::FromAddress(node->address());
      page->add_available_in_medium_free_list(-(*node_size));
    }
  } else if (size_in_bytes <= kLargeListMax) {
    node = large_list_.PickNodeFromList(size_in_bytes, node_size);
    if (node != NULL) {
      DCHECK(size_in_bytes <= *node_size);
      page = Page::FromAddress(node->address());
      page->add_available_in_large_free_list(-(*node_size));
    }
  }

  DCHECK(IsVeryLong() || available() == SumFreeLists());
  return node;
}


// Allocation on the old space free list.  If it succeeds then a new linear
// allocation space has been set up with the top and limit of the space.  If
// the allocation fails then NULL is returned, and the caller can perform a GC
// or allocate a new page before retrying.
HeapObject* FreeList::Allocate(int size_in_bytes) {
  DCHECK(0 < size_in_bytes);
  DCHECK(size_in_bytes <= kMaxBlockSize);
  DCHECK(IsAligned(size_in_bytes, kPointerSize));
  // Don't free list allocate if there is linear space available.
  DCHECK(owner_->limit() - owner_->top() < size_in_bytes);

  int old_linear_size = static_cast<int>(owner_->limit() - owner_->top());
  // Mark the old linear allocation area with a free space map so it can be
  // skipped when scanning the heap.  This also puts it back in the free list
  // if it is big enough.
  owner_->Free(owner_->top(), old_linear_size);

  owner_->heap()->incremental_marking()->OldSpaceStep(size_in_bytes -
                                                      old_linear_size);

  int new_node_size = 0;
  FreeListNode* new_node = FindNodeFor(size_in_bytes, &new_node_size);
  if (new_node == NULL) {
    owner_->SetTopAndLimit(NULL, NULL);
    return NULL;
  }

  int bytes_left = new_node_size - size_in_bytes;
  DCHECK(bytes_left >= 0);

#ifdef DEBUG
  for (int i = 0; i < size_in_bytes / kPointerSize; i++) {
    reinterpret_cast<Object**>(new_node->address())[i] =
        Smi::FromInt(kCodeZapValue);
  }
#endif

  // The old-space-step might have finished sweeping and restarted marking.
  // Verify that it did not turn the page of the new node into an evacuation
  // candidate.
  DCHECK(!MarkCompactCollector::IsOnEvacuationCandidate(new_node));

  const int kThreshold = IncrementalMarking::kAllocatedThreshold;

  // Memory in the linear allocation area is counted as allocated.  We may free
  // a little of this again immediately - see below.
  owner_->Allocate(new_node_size);

  if (owner_->heap()->inline_allocation_disabled()) {
    // Keep the linear allocation area empty if requested to do so, just
    // return area back to the free list instead.
    owner_->Free(new_node->address() + size_in_bytes, bytes_left);
    DCHECK(owner_->top() == NULL && owner_->limit() == NULL);
  } else if (bytes_left > kThreshold &&
             owner_->heap()->incremental_marking()->IsMarkingIncomplete() &&
             FLAG_incremental_marking_steps) {
    int linear_size = owner_->RoundSizeDownToObjectAlignment(kThreshold);
    // We don't want to give too large linear areas to the allocator while
    // incremental marking is going on, because we won't check again whether
    // we want to do another increment until the linear area is used up.
    owner_->Free(new_node->address() + size_in_bytes + linear_size,
                 new_node_size - size_in_bytes - linear_size);
    owner_->SetTopAndLimit(new_node->address() + size_in_bytes,
                           new_node->address() + size_in_bytes + linear_size);
  } else if (bytes_left > 0) {
    // Normally we give the rest of the node to the allocator as its new
    // linear allocation area.
    owner_->SetTopAndLimit(new_node->address() + size_in_bytes,
                           new_node->address() + new_node_size);
  } else {
    // TODO(gc) Try not freeing linear allocation region when bytes_left
    // are zero.
    owner_->SetTopAndLimit(NULL, NULL);
  }

  return new_node;
}


intptr_t FreeList::EvictFreeListItems(Page* p) {
  intptr_t sum = huge_list_.EvictFreeListItemsInList(p);
  p->set_available_in_huge_free_list(0);

  if (sum < p->area_size()) {
    sum += small_list_.EvictFreeListItemsInList(p) +
           medium_list_.EvictFreeListItemsInList(p) +
           large_list_.EvictFreeListItemsInList(p);
    p->set_available_in_small_free_list(0);
    p->set_available_in_medium_free_list(0);
    p->set_available_in_large_free_list(0);
  }

  return sum;
}


bool FreeList::ContainsPageFreeListItems(Page* p) {
  return huge_list_.EvictFreeListItemsInList(p) ||
         small_list_.EvictFreeListItemsInList(p) ||
         medium_list_.EvictFreeListItemsInList(p) ||
         large_list_.EvictFreeListItemsInList(p);
}


void FreeList::RepairLists(Heap* heap) {
  small_list_.RepairFreeList(heap);
  medium_list_.RepairFreeList(heap);
  large_list_.RepairFreeList(heap);
  huge_list_.RepairFreeList(heap);
}


#ifdef DEBUG
intptr_t FreeListCategory::SumFreeList() {
  intptr_t sum = 0;
  FreeListNode* cur = top();
  while (cur != NULL) {
    DCHECK(cur->map() == cur->GetHeap()->raw_unchecked_free_space_map());
    FreeSpace* cur_as_free_space = reinterpret_cast<FreeSpace*>(cur);
    sum += cur_as_free_space->nobarrier_size();
    cur = cur->next();
  }
  return sum;
}


static const int kVeryLongFreeList = 500;


int FreeListCategory::FreeListLength() {
  int length = 0;
  FreeListNode* cur = top();
  while (cur != NULL) {
    length++;
    cur = cur->next();
    if (length == kVeryLongFreeList) return length;
  }
  return length;
}


bool FreeList::IsVeryLong() {
  if (small_list_.FreeListLength() == kVeryLongFreeList) return true;
  if (medium_list_.FreeListLength() == kVeryLongFreeList) return true;
  if (large_list_.FreeListLength() == kVeryLongFreeList) return true;
  if (huge_list_.FreeListLength() == kVeryLongFreeList) return true;
  return false;
}


// This can take a very long time because it is linear in the number of entries
// on the free list, so it should not be called if FreeListLength returns
// kVeryLongFreeList.
intptr_t FreeList::SumFreeLists() {
  intptr_t sum = small_list_.SumFreeList();
  sum += medium_list_.SumFreeList();
  sum += large_list_.SumFreeList();
  sum += huge_list_.SumFreeList();
  return sum;
}
#endif


// -----------------------------------------------------------------------------
// OldSpace implementation

void PagedSpace::PrepareForMarkCompact() {
  // We don't have a linear allocation area while sweeping.  It will be restored
  // on the first allocation after the sweep.
  EmptyAllocationInfo();

  // This counter will be increased for pages which will be swept by the
  // sweeper threads.
  unswept_free_bytes_ = 0;

  // Clear the free list before a full GC---it will be rebuilt afterward.
  free_list_.Reset();
}


intptr_t PagedSpace::SizeOfObjects() {
  DCHECK(!FLAG_concurrent_sweeping ||
         heap()->mark_compact_collector()->sweeping_in_progress() ||
         (unswept_free_bytes_ == 0));
  return Size() - unswept_free_bytes_ - (limit() - top());
}


// After we have booted, we have created a map which represents free space
// on the heap.  If there was already a free list then the elements on it
// were created with the wrong FreeSpaceMap (normally NULL), so we need to
// fix them.
void PagedSpace::RepairFreeListsAfterBoot() { free_list_.RepairLists(heap()); }


void PagedSpace::EvictEvacuationCandidatesFromFreeLists() {
  if (allocation_info_.top() >= allocation_info_.limit()) return;

  if (Page::FromAllocationTop(allocation_info_.top())
          ->IsEvacuationCandidate()) {
    // Create filler object to keep page iterable if it was iterable.
    int remaining =
        static_cast<int>(allocation_info_.limit() - allocation_info_.top());
    heap()->CreateFillerObjectAt(allocation_info_.top(), remaining);

    allocation_info_.set_top(NULL);
    allocation_info_.set_limit(NULL);
  }
}


HeapObject* PagedSpace::WaitForSweeperThreadsAndRetryAllocation(
    int size_in_bytes) {
  MarkCompactCollector* collector = heap()->mark_compact_collector();
  if (collector->sweeping_in_progress()) {
    // Wait for the sweeper threads here and complete the sweeping phase.
    collector->EnsureSweepingCompleted();

    // After waiting for the sweeper threads, there may be new free-list
    // entries.
    return free_list_.Allocate(size_in_bytes);
  }
  return NULL;
}


HeapObject* PagedSpace::SlowAllocateRaw(int size_in_bytes) {
  // Allocation in this space has failed.

  MarkCompactCollector* collector = heap()->mark_compact_collector();
  // Sweeping is still in progress.
  if (collector->sweeping_in_progress()) {
    // First try to refill the free-list, concurrent sweeper threads
    // may have freed some objects in the meantime.
    collector->RefillFreeList(this);

    // Retry the free list allocation.
    HeapObject* object = free_list_.Allocate(size_in_bytes);
    if (object != NULL) return object;

    // If sweeping is still in progress try to sweep pages on the main thread.
    int free_chunk = collector->SweepInParallel(this, size_in_bytes);
    collector->RefillFreeList(this);
    if (free_chunk >= size_in_bytes) {
      HeapObject* object = free_list_.Allocate(size_in_bytes);
      // We should be able to allocate an object here since we just freed that
      // much memory.
      DCHECK(object != NULL);
      if (object != NULL) return object;
    }
  }

  // Free list allocation failed and there is no next page.  Fail if we have
  // hit the old generation size limit that should cause a garbage
  // collection.
  if (!heap()->always_allocate() &&
      heap()->OldGenerationAllocationLimitReached()) {
    // If sweeper threads are active, wait for them at that point and steal
    // elements form their free-lists.
    HeapObject* object = WaitForSweeperThreadsAndRetryAllocation(size_in_bytes);
    if (object != NULL) return object;
  }

  // Try to expand the space and allocate in the new next page.
  if (Expand()) {
    DCHECK(CountTotalPages() > 1 || size_in_bytes <= free_list_.available());
    return free_list_.Allocate(size_in_bytes);
  }

  // If sweeper threads are active, wait for them at that point and steal
  // elements form their free-lists. Allocation may still fail their which
  // would indicate that there is not enough memory for the given allocation.
  return WaitForSweeperThreadsAndRetryAllocation(size_in_bytes);
}


#ifdef DEBUG
void PagedSpace::ReportCodeStatistics(Isolate* isolate) {
  CommentStatistic* comments_statistics =
      isolate->paged_space_comments_statistics();
  ReportCodeKindStatistics(isolate->code_kind_statistics());
  PrintF(
      "Code comment statistics (\"   [ comment-txt   :    size/   "
      "count  (average)\"):\n");
  for (int i = 0; i <= CommentStatistic::kMaxComments; i++) {
    const CommentStatistic& cs = comments_statistics[i];
    if (cs.size > 0) {
      PrintF("   %-30s: %10d/%6d     (%d)\n", cs.comment, cs.size, cs.count,
             cs.size / cs.count);
    }
  }
  PrintF("\n");
}


void PagedSpace::ResetCodeStatistics(Isolate* isolate) {
  CommentStatistic* comments_statistics =
      isolate->paged_space_comments_statistics();
  ClearCodeKindStatistics(isolate->code_kind_statistics());
  for (int i = 0; i < CommentStatistic::kMaxComments; i++) {
    comments_statistics[i].Clear();
  }
  comments_statistics[CommentStatistic::kMaxComments].comment = "Unknown";
  comments_statistics[CommentStatistic::kMaxComments].size = 0;
  comments_statistics[CommentStatistic::kMaxComments].count = 0;
}


// Adds comment to 'comment_statistics' table. Performance OK as long as
// 'kMaxComments' is small
static void EnterComment(Isolate* isolate, const char* comment, int delta) {
  CommentStatistic* comments_statistics =
      isolate->paged_space_comments_statistics();
  // Do not count empty comments
  if (delta <= 0) return;
  CommentStatistic* cs = &comments_statistics[CommentStatistic::kMaxComments];
  // Search for a free or matching entry in 'comments_statistics': 'cs'
  // points to result.
  for (int i = 0; i < CommentStatistic::kMaxComments; i++) {
    if (comments_statistics[i].comment == NULL) {
      cs = &comments_statistics[i];
      cs->comment = comment;
      break;
    } else if (strcmp(comments_statistics[i].comment, comment) == 0) {
      cs = &comments_statistics[i];
      break;
    }
  }
  // Update entry for 'comment'
  cs->size += delta;
  cs->count += 1;
}


// Call for each nested comment start (start marked with '[ xxx', end marked
// with ']'.  RelocIterator 'it' must point to a comment reloc info.
static void CollectCommentStatistics(Isolate* isolate, RelocIterator* it) {
  DCHECK(!it->done());
  DCHECK(it->rinfo()->rmode() == RelocInfo::COMMENT);
  const char* tmp = reinterpret_cast<const char*>(it->rinfo()->data());
  if (tmp[0] != '[') {
    // Not a nested comment; skip
    return;
  }

  // Search for end of nested comment or a new nested comment
  const char* const comment_txt =
      reinterpret_cast<const char*>(it->rinfo()->data());
  const byte* prev_pc = it->rinfo()->pc();
  int flat_delta = 0;
  it->next();
  while (true) {
    // All nested comments must be terminated properly, and therefore exit
    // from loop.
    DCHECK(!it->done());
    if (it->rinfo()->rmode() == RelocInfo::COMMENT) {
      const char* const txt =
          reinterpret_cast<const char*>(it->rinfo()->data());
      flat_delta += static_cast<int>(it->rinfo()->pc() - prev_pc);
      if (txt[0] == ']') break;  // End of nested  comment
      // A new comment
      CollectCommentStatistics(isolate, it);
      // Skip code that was covered with previous comment
      prev_pc = it->rinfo()->pc();
    }
    it->next();
  }
  EnterComment(isolate, comment_txt, flat_delta);
}


// Collects code size statistics:
// - by code kind
// - by code comment
void PagedSpace::CollectCodeStatistics() {
  Isolate* isolate = heap()->isolate();
  HeapObjectIterator obj_it(this);
  for (HeapObject* obj = obj_it.Next(); obj != NULL; obj = obj_it.Next()) {
    if (obj->IsCode()) {
      Code* code = Code::cast(obj);
      isolate->code_kind_statistics()[code->kind()] += code->Size();
      RelocIterator it(code);
      int delta = 0;
      const byte* prev_pc = code->instruction_start();
      while (!it.done()) {
        if (it.rinfo()->rmode() == RelocInfo::COMMENT) {
          delta += static_cast<int>(it.rinfo()->pc() - prev_pc);
          CollectCommentStatistics(isolate, &it);
          prev_pc = it.rinfo()->pc();
        }
        it.next();
      }

      DCHECK(code->instruction_start() <= prev_pc &&
             prev_pc <= code->instruction_end());
      delta += static_cast<int>(code->instruction_end() - prev_pc);
      EnterComment(isolate, "NoComment", delta);
    }
  }
}


void PagedSpace::ReportStatistics() {
  int pct = static_cast<int>(Available() * 100 / Capacity());
  PrintF("  capacity: %" V8_PTR_PREFIX
         "d"
         ", waste: %" V8_PTR_PREFIX
         "d"
         ", available: %" V8_PTR_PREFIX "d, %%%d\n",
         Capacity(), Waste(), Available(), pct);

  if (heap()->mark_compact_collector()->sweeping_in_progress()) {
    heap()->mark_compact_collector()->EnsureSweepingCompleted();
  }
  ClearHistograms(heap()->isolate());
  HeapObjectIterator obj_it(this);
  for (HeapObject* obj = obj_it.Next(); obj != NULL; obj = obj_it.Next())
    CollectHistogramInfo(obj);
  ReportHistogram(heap()->isolate(), true);
}
#endif


// -----------------------------------------------------------------------------
// MapSpace implementation
// TODO(mvstanton): this is weird...the compiler can't make a vtable unless
// there is at least one non-inlined virtual function. I would prefer to hide
// the VerifyObject definition behind VERIFY_HEAP.

void MapSpace::VerifyObject(HeapObject* object) { CHECK(object->IsMap()); }


// -----------------------------------------------------------------------------
// CellSpace and PropertyCellSpace implementation
// TODO(mvstanton): this is weird...the compiler can't make a vtable unless
// there is at least one non-inlined virtual function. I would prefer to hide
// the VerifyObject definition behind VERIFY_HEAP.

void CellSpace::VerifyObject(HeapObject* object) { CHECK(object->IsCell()); }


void PropertyCellSpace::VerifyObject(HeapObject* object) {
  CHECK(object->IsPropertyCell());
}


// -----------------------------------------------------------------------------
// LargeObjectIterator

LargeObjectIterator::LargeObjectIterator(LargeObjectSpace* space) {
  current_ = space->first_page_;
  size_func_ = NULL;
}


LargeObjectIterator::LargeObjectIterator(LargeObjectSpace* space,
                                         HeapObjectCallback size_func) {
  current_ = space->first_page_;
  size_func_ = size_func;
}


HeapObject* LargeObjectIterator::Next() {
  if (current_ == NULL) return NULL;

  HeapObject* object = current_->GetObject();
  current_ = current_->next_page();
  return object;
}


// -----------------------------------------------------------------------------
// LargeObjectSpace
static bool ComparePointers(void* key1, void* key2) { return key1 == key2; }


LargeObjectSpace::LargeObjectSpace(Heap* heap, intptr_t max_capacity,
                                   AllocationSpace id)
    : Space(heap, id, NOT_EXECUTABLE),  // Managed on a per-allocation basis
      max_capacity_(max_capacity),
      first_page_(NULL),
      size_(0),
      page_count_(0),
      objects_size_(0),
      chunk_map_(ComparePointers, 1024) {}


bool LargeObjectSpace::SetUp() {
  first_page_ = NULL;
  size_ = 0;
  maximum_committed_ = 0;
  page_count_ = 0;
  objects_size_ = 0;
  chunk_map_.Clear();
  return true;
}


void LargeObjectSpace::TearDown() {
  while (first_page_ != NULL) {
    LargePage* page = first_page_;
    first_page_ = first_page_->next_page();
    LOG(heap()->isolate(), DeleteEvent("LargeObjectChunk", page->address()));

    ObjectSpace space = static_cast<ObjectSpace>(1 << identity());
    heap()->isolate()->memory_allocator()->PerformAllocationCallback(
        space, kAllocationActionFree, page->size());
    heap()->isolate()->memory_allocator()->Free(page);
  }
  SetUp();
}


AllocationResult LargeObjectSpace::AllocateRaw(int object_size,
                                               Executability executable) {
  // Check if we want to force a GC before growing the old space further.
  // If so, fail the allocation.
  if (!heap()->always_allocate() &&
      heap()->OldGenerationAllocationLimitReached()) {
    return AllocationResult::Retry(identity());
  }

  if (!CanAllocateSize(object_size)) return AllocationResult::Retry(identity());

  LargePage* page = heap()->isolate()->memory_allocator()->AllocateLargePage(
      object_size, this, executable);
  if (page == NULL) return AllocationResult::Retry(identity());
  DCHECK(page->area_size() >= object_size);

  size_ += static_cast<int>(page->size());
  objects_size_ += object_size;
  page_count_++;
  page->set_next_page(first_page_);
  first_page_ = page;

  if (size_ > maximum_committed_) {
    maximum_committed_ = size_;
  }

  // Register all MemoryChunk::kAlignment-aligned chunks covered by
  // this large page in the chunk map.
  uintptr_t base = reinterpret_cast<uintptr_t>(page) / MemoryChunk::kAlignment;
  uintptr_t limit = base + (page->size() - 1) / MemoryChunk::kAlignment;
  for (uintptr_t key = base; key <= limit; key++) {
    HashMap::Entry* entry = chunk_map_.Lookup(reinterpret_cast<void*>(key),
                                              static_cast<uint32_t>(key), true);
    DCHECK(entry != NULL);
    entry->value = page;
  }

  HeapObject* object = page->GetObject();

  MSAN_ALLOCATED_UNINITIALIZED_MEMORY(object->address(), object_size);

  if (Heap::ShouldZapGarbage()) {
    // Make the object consistent so the heap can be verified in OldSpaceStep.
    // We only need to do this in debug builds or if verify_heap is on.
    reinterpret_cast<Object**>(object->address())[0] =
        heap()->fixed_array_map();
    reinterpret_cast<Object**>(object->address())[1] = Smi::FromInt(0);
  }

  heap()->incremental_marking()->OldSpaceStep(object_size);
  return object;
}


size_t LargeObjectSpace::CommittedPhysicalMemory() {
  if (!base::VirtualMemory::HasLazyCommits()) return CommittedMemory();
  size_t size = 0;
  LargePage* current = first_page_;
  while (current != NULL) {
    size += current->CommittedPhysicalMemory();
    current = current->next_page();
  }
  return size;
}


// GC support
Object* LargeObjectSpace::FindObject(Address a) {
  LargePage* page = FindPage(a);
  if (page != NULL) {
    return page->GetObject();
  }
  return Smi::FromInt(0);  // Signaling not found.
}


LargePage* LargeObjectSpace::FindPage(Address a) {
  uintptr_t key = reinterpret_cast<uintptr_t>(a) / MemoryChunk::kAlignment;
  HashMap::Entry* e = chunk_map_.Lookup(reinterpret_cast<void*>(key),
                                        static_cast<uint32_t>(key), false);
  if (e != NULL) {
    DCHECK(e->value != NULL);
    LargePage* page = reinterpret_cast<LargePage*>(e->value);
    DCHECK(page->is_valid());
    if (page->Contains(a)) {
      return page;
    }
  }
  return NULL;
}


void LargeObjectSpace::FreeUnmarkedObjects() {
  LargePage* previous = NULL;
  LargePage* current = first_page_;
  while (current != NULL) {
    HeapObject* object = current->GetObject();
    // Can this large page contain pointers to non-trivial objects.  No other
    // pointer object is this big.
    bool is_pointer_object = object->IsFixedArray();
    MarkBit mark_bit = Marking::MarkBitFrom(object);
    if (mark_bit.Get()) {
      mark_bit.Clear();
      Page::FromAddress(object->address())->ResetProgressBar();
      Page::FromAddress(object->address())->ResetLiveBytes();
      previous = current;
      current = current->next_page();
    } else {
      LargePage* page = current;
      // Cut the chunk out from the chunk list.
      current = current->next_page();
      if (previous == NULL) {
        first_page_ = current;
      } else {
        previous->set_next_page(current);
      }

      // Free the chunk.
      heap()->mark_compact_collector()->ReportDeleteIfNeeded(object,
                                                             heap()->isolate());
      size_ -= static_cast<int>(page->size());
      objects_size_ -= object->Size();
      page_count_--;

      // Remove entries belonging to this page.
      // Use variable alignment to help pass length check (<= 80 characters)
      // of single line in tools/presubmit.py.
      const intptr_t alignment = MemoryChunk::kAlignment;
      uintptr_t base = reinterpret_cast<uintptr_t>(page) / alignment;
      uintptr_t limit = base + (page->size() - 1) / alignment;
      for (uintptr_t key = base; key <= limit; key++) {
        chunk_map_.Remove(reinterpret_cast<void*>(key),
                          static_cast<uint32_t>(key));
      }

      if (is_pointer_object) {
        heap()->QueueMemoryChunkForFree(page);
      } else {
        heap()->isolate()->memory_allocator()->Free(page);
      }
    }
  }
  heap()->FreeQueuedChunks();
}


bool LargeObjectSpace::Contains(HeapObject* object) {
  Address address = object->address();
  MemoryChunk* chunk = MemoryChunk::FromAddress(address);

  bool owned = (chunk->owner() == this);

  SLOW_DCHECK(!owned || FindObject(address)->IsHeapObject());

  return owned;
}


#ifdef VERIFY_HEAP
// We do not assume that the large object iterator works, because it depends
// on the invariants we are checking during verification.
void LargeObjectSpace::Verify() {
  for (LargePage* chunk = first_page_; chunk != NULL;
       chunk = chunk->next_page()) {
    // Each chunk contains an object that starts at the large object page's
    // object area start.
    HeapObject* object = chunk->GetObject();
    Page* page = Page::FromAddress(object->address());
    CHECK(object->address() == page->area_start());

    // The first word should be a map, and we expect all map pointers to be
    // in map space.
    Map* map = object->map();
    CHECK(map->IsMap());
    CHECK(heap()->map_space()->Contains(map));

    // We have only code, sequential strings, external strings
    // (sequential strings that have been morphed into external
    // strings), fixed arrays, byte arrays, and constant pool arrays in the
    // large object space.
    CHECK(object->IsCode() || object->IsSeqString() ||
          object->IsExternalString() || object->IsFixedArray() ||
          object->IsFixedDoubleArray() || object->IsByteArray() ||
          object->IsConstantPoolArray());

    // The object itself should look OK.
    object->ObjectVerify();

    // Byte arrays and strings don't have interior pointers.
    if (object->IsCode()) {
      VerifyPointersVisitor code_visitor;
      object->IterateBody(map->instance_type(), object->Size(), &code_visitor);
    } else if (object->IsFixedArray()) {
      FixedArray* array = FixedArray::cast(object);
      for (int j = 0; j < array->length(); j++) {
        Object* element = array->get(j);
        if (element->IsHeapObject()) {
          HeapObject* element_object = HeapObject::cast(element);
          CHECK(heap()->Contains(element_object));
          CHECK(element_object->map()->IsMap());
        }
      }
    }
  }
}
#endif


#ifdef DEBUG
void LargeObjectSpace::Print() {
  OFStream os(stdout);
  LargeObjectIterator it(this);
  for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
    obj->Print(os);
  }
}


void LargeObjectSpace::ReportStatistics() {
  PrintF("  size: %" V8_PTR_PREFIX "d\n", size_);
  int num_objects = 0;
  ClearHistograms(heap()->isolate());
  LargeObjectIterator it(this);
  for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) {
    num_objects++;
    CollectHistogramInfo(obj);
  }

  PrintF(
      "  number of objects %d, "
      "size of objects %" V8_PTR_PREFIX "d\n",
      num_objects, objects_size_);
  if (num_objects > 0) ReportHistogram(heap()->isolate(), false);
}


void LargeObjectSpace::CollectCodeStatistics() {
  Isolate* isolate = heap()->isolate();
  LargeObjectIterator obj_it(this);
  for (HeapObject* obj = obj_it.Next(); obj != NULL; obj = obj_it.Next()) {
    if (obj->IsCode()) {
      Code* code = Code::cast(obj);
      isolate->code_kind_statistics()[code->kind()] += code->Size();
    }
  }
}


void Page::Print() {
  // Make a best-effort to print the objects in the page.
  PrintF("Page@%p in %s\n", this->address(),
         AllocationSpaceName(this->owner()->identity()));
  printf(" --------------------------------------\n");
  HeapObjectIterator objects(this, heap()->GcSafeSizeOfOldObjectFunction());
  unsigned mark_size = 0;
  for (HeapObject* object = objects.Next(); object != NULL;
       object = objects.Next()) {
    bool is_marked = Marking::MarkBitFrom(object).Get();
    PrintF(" %c ", (is_marked ? '!' : ' '));  // Indent a little.
    if (is_marked) {
      mark_size += heap()->GcSafeSizeOfOldObjectFunction()(object);
    }
    object->ShortPrint();
    PrintF("\n");
  }
  printf(" --------------------------------------\n");
  printf(" Marked: %x, LiveCount: %x\n", mark_size, LiveBytes());
}

#endif  // DEBUG
}
}  // namespace v8::internal