Reserve boot image memory in one go.

Load boot image components into the reserved memory.

Test: m test-art-host-gtest
Test: testrunner.py --host
Test: Pixel 2 XL boots.
Test: m test-art-target-gtest
Test: testrunner.py --target --optimizing
Bug: 77856493
Change-Id: I214f947979bc0bbfc6df4312527504e90b88a01d
diff --git a/runtime/dexopt_test.cc b/runtime/dexopt_test.cc
index 127e14e..93af77f 100644
--- a/runtime/dexopt_test.cc
+++ b/runtime/dexopt_test.cc
@@ -151,11 +151,11 @@
   std::unique_ptr<OatFile> odex_file(OatFile::Open(/* zip_fd */ -1,
                                                    oat_location.c_str(),
                                                    oat_location.c_str(),
-                                                   nullptr,
-                                                   nullptr,
-                                                   false,
-                                                   /*low_4gb*/false,
+                                                   /* requested_base */ nullptr,
+                                                   /* executable */ false,
+                                                   /* low_4gb */ false,
                                                    dex_location.c_str(),
+                                                   /* reservation */ nullptr,
                                                    &error_msg));
   ASSERT_TRUE(odex_file.get() != nullptr) << error_msg;
   EXPECT_EQ(pic, odex_file->IsPic());
diff --git a/runtime/elf_file.cc b/runtime/elf_file.cc
index d45a689..e7715c4 100644
--- a/runtime/elf_file.cc
+++ b/runtime/elf_file.cc
@@ -37,9 +37,7 @@
 using android::base::StringPrintf;
 
 template <typename ElfTypes>
-ElfFileImpl<ElfTypes>::ElfFileImpl(File* file, bool writable,
-                                   bool program_header_only,
-                                   uint8_t* requested_base)
+ElfFileImpl<ElfTypes>::ElfFileImpl(File* file, bool writable, bool program_header_only)
   : writable_(writable),
     program_header_only_(program_header_only),
     header_(nullptr),
@@ -54,8 +52,7 @@
     dynstr_section_start_(nullptr),
     hash_section_start_(nullptr),
     symtab_symbol_table_(nullptr),
-    dynsym_symbol_table_(nullptr),
-    requested_base_(requested_base) {
+    dynsym_symbol_table_(nullptr) {
   CHECK(file != nullptr);
 }
 
@@ -64,10 +61,9 @@
                                                    bool writable,
                                                    bool program_header_only,
                                                    bool low_4gb,
-                                                   std::string* error_msg,
-                                                   uint8_t* requested_base) {
-  std::unique_ptr<ElfFileImpl<ElfTypes>> elf_file(new ElfFileImpl<ElfTypes>
-      (file, writable, program_header_only, requested_base));
+                                                   std::string* error_msg) {
+  std::unique_ptr<ElfFileImpl<ElfTypes>> elf_file(
+      new ElfFileImpl<ElfTypes>(file, writable, program_header_only));
   int prot;
   int flags;
   if (writable) {
@@ -89,9 +85,8 @@
                                                    int flags,
                                                    bool low_4gb,
                                                    std::string* error_msg) {
-  std::unique_ptr<ElfFileImpl<ElfTypes>> elf_file(new ElfFileImpl<ElfTypes>
-      (file, (prot & PROT_WRITE) == PROT_WRITE, /*program_header_only*/false,
-      /*requested_base*/nullptr));
+  std::unique_ptr<ElfFileImpl<ElfTypes>> elf_file(
+      new ElfFileImpl<ElfTypes>(file, (prot & PROT_WRITE) != 0, /* program_header_only */ false));
   if (!elf_file->Setup(file, prot, flags, low_4gb, error_msg)) {
     return nullptr;
   }
@@ -684,9 +679,7 @@
 typename ElfTypes::Phdr* ElfFileImpl<ElfTypes>::GetProgramHeader(Elf_Word i) const {
   CHECK_LT(i, GetProgramHeaderNum()) << file_path_;  // Sanity check for caller.
   uint8_t* program_header = GetProgramHeadersStart() + (i * GetHeader().e_phentsize);
-  if (program_header >= End()) {
-    return nullptr;  // Failure condition.
-  }
+  CHECK_LT(program_header, End());
   return reinterpret_cast<Elf_Phdr*>(program_header);
 }
 
@@ -1027,9 +1020,17 @@
   return *(GetRelaSectionStart(section_header) + i);
 }
 
-// Base on bionic phdr_table_get_load_size
 template <typename ElfTypes>
 bool ElfFileImpl<ElfTypes>::GetLoadedSize(size_t* size, std::string* error_msg) const {
+  uint8_t* vaddr_begin;
+  return GetLoadedAddressRange(&vaddr_begin, size, error_msg);
+}
+
+// Base on bionic phdr_table_get_load_size
+template <typename ElfTypes>
+bool ElfFileImpl<ElfTypes>::GetLoadedAddressRange(/*out*/uint8_t** vaddr_begin,
+                                                  /*out*/size_t* vaddr_size,
+                                                  /*out*/std::string* error_msg) const {
   Elf_Addr min_vaddr = static_cast<Elf_Addr>(-1);
   Elf_Addr max_vaddr = 0u;
   for (Elf_Word i = 0; i < GetProgramHeaderNum(); i++) {
@@ -1048,7 +1049,8 @@
           << program_header->p_vaddr << "+0x" << program_header->p_memsz << "=0x" << end_vaddr
           << " in ELF file \"" << file_path_ << "\"";
       *error_msg = oss.str();
-      *size = static_cast<size_t>(-1);
+      *vaddr_begin = nullptr;
+      *vaddr_size = static_cast<size_t>(-1);
       return false;
     }
     if (end_vaddr > max_vaddr) {
@@ -1058,17 +1060,19 @@
   min_vaddr = RoundDown(min_vaddr, kPageSize);
   max_vaddr = RoundUp(max_vaddr, kPageSize);
   CHECK_LT(min_vaddr, max_vaddr) << file_path_;
-  Elf_Addr loaded_size = max_vaddr - min_vaddr;
-  // Check that the loaded_size fits in size_t.
-  if (UNLIKELY(loaded_size > std::numeric_limits<size_t>::max())) {
+  // Check that the range fits into the runtime address space.
+  if (UNLIKELY(max_vaddr - 1u > std::numeric_limits<size_t>::max())) {
     std::ostringstream oss;
-    oss << "Loaded size is 0x" << std::hex << loaded_size << " but maximum size_t is 0x"
-        << std::numeric_limits<size_t>::max() << " for ELF file \"" << file_path_ << "\"";
+    oss << "Loaded range is 0x" << std::hex << min_vaddr << "-0x" << max_vaddr
+        << " but maximum size_t is 0x" << std::numeric_limits<size_t>::max()
+        << " for ELF file \"" << file_path_ << "\"";
     *error_msg = oss.str();
-    *size = static_cast<size_t>(-1);
+    *vaddr_begin = nullptr;
+    *vaddr_size = static_cast<size_t>(-1);
     return false;
   }
-  *size = loaded_size;
+  *vaddr_begin = reinterpret_cast<uint8_t*>(min_vaddr);
+  *vaddr_size = dchecked_integral_cast<size_t>(max_vaddr - min_vaddr);
   return true;
 }
 
@@ -1099,7 +1103,8 @@
 bool ElfFileImpl<ElfTypes>::Load(File* file,
                                  bool executable,
                                  bool low_4gb,
-                                 std::string* error_msg) {
+                                 /*inout*/MemMap* reservation,
+                                 /*out*/std::string* error_msg) {
   CHECK(program_header_only_) << file->GetPath();
 
   if (executable) {
@@ -1115,11 +1120,6 @@
   bool reserved = false;
   for (Elf_Word i = 0; i < GetProgramHeaderNum(); i++) {
     Elf_Phdr* program_header = GetProgramHeader(i);
-    if (program_header == nullptr) {
-      *error_msg = StringPrintf("No program header for entry %d in ELF file %s.",
-                                i, file->GetPath().c_str());
-      return false;
-    }
 
     // Record .dynamic header information for later use
     if (program_header->p_type == PT_DYNAMIC) {
@@ -1150,41 +1150,39 @@
     }
     size_t file_length = static_cast<size_t>(temp_file_length);
     if (!reserved) {
-      uint8_t* reserve_base = reinterpret_cast<uint8_t*>(program_header->p_vaddr);
-      uint8_t* reserve_base_override = reserve_base;
-      // Override the base (e.g. when compiling with --compile-pic)
-      if (requested_base_ != nullptr) {
-        reserve_base_override = requested_base_;
-      }
-      std::string reservation_name("ElfFile reservation for ");
-      reservation_name += file->GetPath();
-      size_t loaded_size;
-      if (!GetLoadedSize(&loaded_size, error_msg)) {
+      uint8_t* vaddr_begin;
+      size_t vaddr_size;
+      if (!GetLoadedAddressRange(&vaddr_begin, &vaddr_size, error_msg)) {
         DCHECK(!error_msg->empty());
         return false;
       }
-      MemMap reserve = MemMap::MapAnonymous(reservation_name.c_str(),
-                                            reserve_base_override,
-                                            loaded_size,
-                                            PROT_NONE,
-                                            low_4gb,
-                                            error_msg);
-      if (!reserve.IsValid()) {
+      std::string reservation_name = "ElfFile reservation for " + file->GetPath();
+      MemMap local_reservation = MemMap::MapAnonymous(
+          reservation_name.c_str(),
+          (reservation != nullptr) ? reservation->Begin() : nullptr,
+          vaddr_size,
+          PROT_NONE,
+          low_4gb,
+          /* reuse */ false,
+          reservation,
+          error_msg);
+      if (!local_reservation.IsValid()) {
         *error_msg = StringPrintf("Failed to allocate %s: %s",
-                                  reservation_name.c_str(), error_msg->c_str());
+                                  reservation_name.c_str(),
+                                  error_msg->c_str());
         return false;
       }
       reserved = true;
 
-      // Base address is the difference of actual mapped location and the p_vaddr
-      base_address_ = reinterpret_cast<uint8_t*>(reinterpret_cast<uintptr_t>(reserve.Begin())
-                       - reinterpret_cast<uintptr_t>(reserve_base));
+      // Base address is the difference of actual mapped location and the vaddr_begin.
+      base_address_ = reinterpret_cast<uint8_t*>(
+          static_cast<uintptr_t>(local_reservation.Begin() - vaddr_begin));
       // By adding the p_vaddr of a section/symbol to base_address_ we will always get the
       // dynamic memory address of where that object is actually mapped
       //
       // TODO: base_address_ needs to be calculated in ::Open, otherwise
       // FindDynamicSymbolAddress returns the wrong values until Load is called.
-      segments_.push_back(std::move(reserve));
+      segments_.push_back(std::move(local_reservation));
     }
     // empty segment, nothing to map
     if (program_header->p_memsz == 0) {
@@ -1239,9 +1237,10 @@
                                    flags,
                                    file->Fd(),
                                    program_header->p_offset,
-                                   /*low4_gb*/false,
-                                   /*reuse*/true,  // implies MAP_FIXED
+                                   /* low4_gb */ false,
                                    file->GetPath().c_str(),
+                                   /* reuse */ true,  // implies MAP_FIXED
+                                   /* reservation */ nullptr,
                                    error_msg);
       if (!segment.IsValid()) {
         *error_msg = StringPrintf("Failed to map ELF file segment %d from %s: %s",
@@ -1265,6 +1264,7 @@
                                             prot,
                                             /* low_4gb */ false,
                                             /* reuse */ true,
+                                            /* reservation */ nullptr,
                                             error_msg);
       if (!segment.IsValid()) {
         *error_msg = StringPrintf("Failed to map zero-initialized ELF file segment %d from %s: %s",
@@ -1704,8 +1704,7 @@
                        bool writable,
                        bool program_header_only,
                        bool low_4gb,
-                       std::string* error_msg,
-                       uint8_t* requested_base) {
+                       /*out*/std::string* error_msg) {
   if (file->GetLength() < EI_NIDENT) {
     *error_msg = StringPrintf("File %s is too short to be a valid ELF file",
                               file->GetPath().c_str());
@@ -1728,8 +1727,7 @@
                                                        writable,
                                                        program_header_only,
                                                        low_4gb,
-                                                       error_msg,
-                                                       requested_base);
+                                                       error_msg);
     if (elf_file_impl == nullptr) {
       return nullptr;
     }
@@ -1739,8 +1737,7 @@
                                                        writable,
                                                        program_header_only,
                                                        low_4gb,
-                                                       error_msg,
-                                                       requested_base);
+                                                       error_msg);
     if (elf_file_impl == nullptr) {
       return nullptr;
     }
@@ -1754,7 +1751,7 @@
   }
 }
 
-ElfFile* ElfFile::Open(File* file, int mmap_prot, int mmap_flags, std::string* error_msg) {
+ElfFile* ElfFile::Open(File* file, int mmap_prot, int mmap_flags, /*out*/std::string* error_msg) {
   // low_4gb support not required for this path.
   constexpr bool low_4gb = false;
   if (file->GetLength() < EI_NIDENT) {
@@ -1811,8 +1808,12 @@
     return elf32_->func(__VA_ARGS__); \
   }
 
-bool ElfFile::Load(File* file, bool executable, bool low_4gb, std::string* error_msg) {
-  DELEGATE_TO_IMPL(Load, file, executable, low_4gb, error_msg);
+bool ElfFile::Load(File* file,
+                   bool executable,
+                   bool low_4gb,
+                   /*inout*/MemMap* reservation,
+                   /*out*/std::string* error_msg) {
+  DELEGATE_TO_IMPL(Load, file, executable, low_4gb, reservation, error_msg);
 }
 
 const uint8_t* ElfFile::FindDynamicSymbolAddress(const std::string& symbol_name) const {
diff --git a/runtime/elf_file.h b/runtime/elf_file.h
index ab9e6fa..8da7e1a 100644
--- a/runtime/elf_file.h
+++ b/runtime/elf_file.h
@@ -26,6 +26,9 @@
 #include "./elf.h"
 
 namespace art {
+
+class MemMap;
+
 template <typename ElfTypes>
 class ElfFileImpl;
 
@@ -42,18 +45,21 @@
                        bool writable,
                        bool program_header_only,
                        bool low_4gb,
-                       std::string* error_msg,
-                       uint8_t* requested_base = nullptr);  // TODO: move arg to before error_msg.
+                       /*out*/std::string* error_msg);
   // Open with specific mmap flags, Always maps in the whole file, not just the
   // program header sections.
   static ElfFile* Open(File* file,
                        int mmap_prot,
                        int mmap_flags,
-                       std::string* error_msg);
+                       /*out*/std::string* error_msg);
   ~ElfFile();
 
   // Load segments into memory based on PT_LOAD program headers
-  bool Load(File* file, bool executable, bool low_4gb, std::string* error_msg);
+  bool Load(File* file,
+            bool executable,
+            bool low_4gb,
+            /*inout*/MemMap* reservation,
+            /*out*/std::string* error_msg);
 
   const uint8_t* FindDynamicSymbolAddress(const std::string& symbol_name) const;
 
diff --git a/runtime/elf_file_impl.h b/runtime/elf_file_impl.h
index 58c38a4..b55b60f 100644
--- a/runtime/elf_file_impl.h
+++ b/runtime/elf_file_impl.h
@@ -48,13 +48,12 @@
                            bool writable,
                            bool program_header_only,
                            bool low_4gb,
-                           std::string* error_msg,
-                           uint8_t* requested_base = nullptr);
+                           /*out*/std::string* error_msg);
   static ElfFileImpl* Open(File* file,
                            int mmap_prot,
                            int mmap_flags,
                            bool low_4gb,
-                           std::string* error_msg);
+                           /*out*/std::string* error_msg);
   ~ElfFileImpl();
 
   const std::string& GetFilePath() const {
@@ -115,7 +114,11 @@
 
   // Load segments into memory based on PT_LOAD program headers.
   // executable is true at run time, false at compile time.
-  bool Load(File* file, bool executable, bool low_4gb, std::string* error_msg);
+  bool Load(File* file,
+            bool executable,
+            bool low_4gb,
+            /*inout*/MemMap* reservation,
+            /*out*/std::string* error_msg);
 
   bool Fixup(Elf_Addr base_address);
   bool FixupDynamic(Elf_Addr base_address);
@@ -131,7 +134,11 @@
   bool Strip(File* file, std::string* error_msg);
 
  private:
-  ElfFileImpl(File* file, bool writable, bool program_header_only, uint8_t* requested_base);
+  ElfFileImpl(File* file, bool writable, bool program_header_only);
+
+  bool GetLoadedAddressRange(/*out*/uint8_t** vaddr_begin,
+                             /*out*/size_t* vaddr_size,
+                             /*out*/std::string* error_msg) const;
 
   bool Setup(File* file, int prot, int flags, bool low_4gb, std::string* error_msg);
 
@@ -217,9 +224,6 @@
   SymbolTable* symtab_symbol_table_;
   SymbolTable* dynsym_symbol_table_;
 
-  // Override the 'base' p_vaddr in the first LOAD segment with this value (if non-null).
-  uint8_t* requested_base_;
-
   DISALLOW_COPY_AND_ASSIGN(ElfFileImpl);
 };
 
diff --git a/runtime/gc/collector/immune_spaces_test.cc b/runtime/gc/collector/immune_spaces_test.cc
index 677e3f3..7bd87bd 100644
--- a/runtime/gc/collector/immune_spaces_test.cc
+++ b/runtime/gc/collector/immune_spaces_test.cc
@@ -41,13 +41,13 @@
 class DummyImageSpace : public space::ImageSpace {
  public:
   DummyImageSpace(MemMap&& map,
-                  accounting::ContinuousSpaceBitmap* live_bitmap,
+                  std::unique_ptr<accounting::ContinuousSpaceBitmap> live_bitmap,
                   std::unique_ptr<DummyOatFile>&& oat_file,
                   MemMap&& oat_map)
       : ImageSpace("DummyImageSpace",
                    /*image_location*/"",
                    std::move(map),
-                   live_bitmap,
+                   std::move(live_bitmap),
                    map.End()),
         oat_map_(std::move(oat_map)) {
     oat_file_ = std::move(oat_file);
@@ -130,7 +130,7 @@
         ImageHeader::kStorageModeUncompressed,
         /*storage_size*/0u);
     return new DummyImageSpace(std::move(map),
-                               live_bitmap.release(),
+                               std::move(live_bitmap),
                                std::move(oat_file),
                                std::move(oat_map));
   }
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index 7178627..0af049a 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -63,7 +63,7 @@
 ImageSpace::ImageSpace(const std::string& image_filename,
                        const char* image_location,
                        MemMap&& mem_map,
-                       accounting::ContinuousSpaceBitmap* live_bitmap,
+                       std::unique_ptr<accounting::ContinuousSpaceBitmap> live_bitmap,
                        uint8_t* end)
     : MemMapSpace(image_filename,
                   std::move(mem_map),
@@ -71,10 +71,10 @@
                   end,
                   end,
                   kGcRetentionPolicyNeverCollect),
+      live_bitmap_(std::move(live_bitmap)),
       oat_file_non_owned_(nullptr),
       image_location_(image_location) {
-  DCHECK(live_bitmap != nullptr);
-  live_bitmap_.reset(live_bitmap);
+  DCHECK(live_bitmap_ != nullptr);
 }
 
 static int32_t ChooseRelocationOffsetDelta(int32_t min_delta, int32_t max_delta) {
@@ -480,52 +480,16 @@
 }
 
 // Helper class encapsulating loading, so we can access private ImageSpace members (this is a
-// friend class), but not declare functions in the header.
+// nested class), but not declare functions in the header.
 class ImageSpace::Loader {
  public:
-  static std::unique_ptr<ImageSpace> Load(const std::string& image_location,
-                                          const std::string& image_filename,
-                                          bool is_zygote,
-                                          bool is_global_cache,
-                                          bool validate_oat_file,
-                                          std::string* error_msg)
-      REQUIRES_SHARED(Locks::mutator_lock_) {
-    // Should this be a RDWR lock? This is only a defensive measure, as at
-    // this point the image should exist.
-    // However, only the zygote can write into the global dalvik-cache, so
-    // restrict to zygote processes, or any process that isn't using
-    // /data/dalvik-cache (which we assume to be allowed to write there).
-    const bool rw_lock = is_zygote || !is_global_cache;
-
-    // Note that we must not use the file descriptor associated with
-    // ScopedFlock::GetFile to Init the image file. We want the file
-    // descriptor (and the associated exclusive lock) to be released when
-    // we leave Create.
-    ScopedFlock image = LockedFile::Open(image_filename.c_str(),
-                                         rw_lock ? (O_CREAT | O_RDWR) : O_RDONLY /* flags */,
-                                         true /* block */,
-                                         error_msg);
-
-    VLOG(startup) << "Using image file " << image_filename.c_str() << " for image location "
-                  << image_location;
-    // If we are in /system we can assume the image is good. We can also
-    // assume this if we are using a relocated image (i.e. image checksum
-    // matches) since this is only different by the offset. We need this to
-    // make sure that host tests continue to work.
-    // Since we are the boot image, pass null since we load the oat file from the boot image oat
-    // file name.
-    return Init(image_filename.c_str(),
-                image_location.c_str(),
-                validate_oat_file,
-                /* oat_file */nullptr,
-                error_msg);
-  }
-
   static std::unique_ptr<ImageSpace> Init(const char* image_filename,
                                           const char* image_location,
                                           bool validate_oat_file,
                                           const OatFile* oat_file,
-                                          std::string* error_msg)
+                                          /*inout*/MemMap* image_reservation,
+                                          /*inout*/MemMap* oat_reservation,
+                                          /*out*/std::string* error_msg)
       REQUIRES_SHARED(Locks::mutator_lock_) {
     CHECK(image_filename != nullptr);
     CHECK(image_location != nullptr);
@@ -616,22 +580,27 @@
     // image at the image begin, the amount of fixup work required is minimized.
     // If it is pic we will retry with error_msg for the failure case. Pass a null error_msg to
     // avoid reading proc maps for a mapping failure and slowing everything down.
-    map = LoadImageFile(image_filename,
-                        image_location,
-                        *image_header,
-                        image_header->GetImageBegin(),
-                        file->Fd(),
-                        logger,
-                        image_header->IsPic() ? nullptr : error_msg);
+    // For the boot image, we have already reserved the memory and we load the image
+    // into the `image_reservation`.
+    map = LoadImageFile(
+        image_filename,
+        image_location,
+        *image_header,
+        image_header->GetImageBegin(),
+        file->Fd(),
+        logger,
+        image_reservation,
+        (image_reservation == nullptr && image_header->IsPic()) ? nullptr : error_msg);
     // If the header specifies PIC mode, we can also map at a random low_4gb address since we can
     // relocate in-place.
-    if (!map.IsValid() && image_header->IsPic()) {
+    if (!map.IsValid() && image_reservation == nullptr && image_header->IsPic()) {
       map = LoadImageFile(image_filename,
                           image_location,
                           *image_header,
                           /* address */ nullptr,
                           file->Fd(),
                           logger,
+                          /* image_reservation */ nullptr,
                           error_msg);
     }
     // Were we able to load something and continue?
@@ -641,15 +610,13 @@
     }
     DCHECK_EQ(0, memcmp(image_header, map.Begin(), sizeof(ImageHeader)));
 
-    MemMap image_bitmap_map = MemMap::MapFileAtAddress(nullptr,
-                                                       bitmap_section.Size(),
-                                                       PROT_READ, MAP_PRIVATE,
-                                                       file->Fd(),
-                                                       image_bitmap_offset,
-                                                       /*low_4gb*/false,
-                                                       /*reuse*/false,
-                                                       image_filename,
-                                                       error_msg);
+    MemMap image_bitmap_map = MemMap::MapFile(bitmap_section.Size(),
+                                              PROT_READ, MAP_PRIVATE,
+                                              file->Fd(),
+                                              image_bitmap_offset,
+                                              /* low_4gb */ false,
+                                              image_filename,
+                                              error_msg);
     if (!image_bitmap_map.IsValid()) {
       *error_msg = StringPrintf("Failed to map image bitmap: %s", error_msg->c_str());
       return nullptr;
@@ -694,7 +661,7 @@
     std::unique_ptr<ImageSpace> space(new ImageSpace(image_filename,
                                                      image_location,
                                                      std::move(map),
-                                                     bitmap.release(),
+                                                     std::move(bitmap),
                                                      image_end));
 
     // VerifyImageAllocations() will be called later in Runtime::Init()
@@ -704,7 +671,7 @@
     // set yet at this point.
     if (oat_file == nullptr) {
       TimingLogger::ScopedTiming timing("OpenOatFile", &logger);
-      space->oat_file_ = OpenOatFile(*space, image_filename, error_msg);
+      space->oat_file_ = OpenOatFile(*space, image_filename, oat_reservation, error_msg);
       if (space->oat_file_ == nullptr) {
         DCHECK(!error_msg->empty());
         return nullptr;
@@ -787,7 +754,8 @@
                               uint8_t* address,
                               int fd,
                               TimingLogger& logger,
-                              std::string* error_msg) {
+                              /*inout*/MemMap* image_reservation,
+                              /*out*/std::string* error_msg) {
     TimingLogger::ScopedTiming timing("MapImageFile", &logger);
     const ImageHeader::StorageMode storage_mode = image_header.GetStorageMode();
     if (storage_mode == ImageHeader::kStorageModeUncompressed) {
@@ -796,10 +764,11 @@
                                       PROT_READ | PROT_WRITE,
                                       MAP_PRIVATE,
                                       fd,
-                                      0,
-                                      /*low_4gb*/true,
-                                      /*reuse*/false,
+                                      /* start */ 0,
+                                      /* low_4gb */ true,
                                       image_filename,
+                                      /* reuse */ false,
+                                      image_reservation,
                                       error_msg);
     }
 
@@ -817,7 +786,9 @@
                                       address,
                                       image_header.GetImageSize(),
                                       PROT_READ | PROT_WRITE,
-                                      /*low_4gb*/ true,
+                                      /* low_4gb */ true,
+                                      /* reuse */ false,
+                                      image_reservation,
                                       error_msg);
     if (map.IsValid()) {
       const size_t stored_size = image_header.GetDataSize();
@@ -826,8 +797,8 @@
                                         PROT_READ,
                                         MAP_PRIVATE,
                                         fd,
-                                        /*offset*/0,
-                                        /*low_4gb*/false,
+                                        /* offset */ 0,
+                                        /* low_4gb */ false,
                                         image_filename,
                                         error_msg);
       if (!temp_map.IsValid()) {
@@ -1396,6 +1367,7 @@
 
   static std::unique_ptr<OatFile> OpenOatFile(const ImageSpace& image,
                                               const char* image_path,
+                                              /*inout*/MemMap* oat_reservation,
                                               std::string* error_msg) {
     const ImageHeader& image_header = image.GetImageHeader();
     std::string oat_filename = ImageHeader::GetOatLocationFromImageLocation(image_path);
@@ -1406,10 +1378,10 @@
                                                     oat_filename,
                                                     oat_filename,
                                                     image_header.GetOatDataBegin(),
-                                                    image_header.GetOatFileBegin(),
                                                     !Runtime::Current()->IsAotCompiler(),
-                                                    /*low_4gb*/false,
-                                                    nullptr,
+                                                    /* low_4gb */ false,
+                                                    /* abs_dex_location */ nullptr,
+                                                    oat_reservation,
                                                     error_msg));
     if (oat_file == nullptr) {
       *error_msg = StringPrintf("Failed to open oat file '%s' referenced from image %s: %s",
@@ -1497,20 +1469,48 @@
     if (!GetBootClassPathImageLocations(image_location_, filename, &locations, error_msg)) {
       return false;
     }
+    uint32_t image_start;
+    uint32_t image_end;
+    uint32_t oat_end;
+    if (!GetBootImageAddressRange(filename, &image_start, &image_end, &oat_end, error_msg)) {
+      return false;
+    }
+    if (locations.size() > 1u) {
+      std::string last_filename = GetSystemImageFilename(locations.back().c_str(), image_isa_);
+      uint32_t dummy;
+      if (!GetBootImageAddressRange(last_filename, &dummy, &image_end, &oat_end, error_msg)) {
+        return false;
+      }
+    }
+    MemMap image_reservation;
+    MemMap oat_reservation;
+    if (!ReserveBootImageMemory(image_start,
+                                image_end,
+                                oat_end,
+                                &image_reservation,
+                                &oat_reservation,
+                                error_msg)) {
+      return false;
+    }
+
     std::vector<std::unique_ptr<ImageSpace>> spaces;
     spaces.reserve(locations.size());
     for (const std::string& location : locations) {
       filename = GetSystemImageFilename(location.c_str(), image_isa_);
-      spaces.push_back(Loader::Load(location,
-                                    filename,
-                                    is_zygote_,
-                                    is_global_cache_,
-                                    /* validate_oat_file */ false,
-                                    error_msg));
+      spaces.push_back(Load(location,
+                            filename,
+                            /* validate_oat_file */ false,
+                            &image_reservation,
+                            &oat_reservation,
+                            error_msg));
       if (spaces.back() == nullptr) {
         return false;
       }
     }
+    if (!CheckReservationsExhausted(image_reservation, oat_reservation, error_msg)) {
+      return false;
+    }
+
     *oat_file_end = GetOatFileEnd(spaces);
     boot_image_spaces->swap(spaces);
     return true;
@@ -1527,6 +1527,36 @@
     if (!GetBootClassPathImageLocations(image_location_, cache_filename_, &locations, error_msg)) {
       return false;
     }
+    uint32_t image_start;
+    uint32_t image_end;
+    uint32_t oat_end;
+    if (!GetBootImageAddressRange(cache_filename_, &image_start, &image_end, &oat_end, error_msg)) {
+      return false;
+    }
+    if (locations.size() > 1u) {
+      std::string last_filename;
+      if (!GetDalvikCacheFilename(locations.back().c_str(),
+                                  dalvik_cache_.c_str(),
+                                  &last_filename,
+                                  error_msg)) {
+        return false;
+      }
+      uint32_t dummy;
+      if (!GetBootImageAddressRange(last_filename, &dummy, &image_end, &oat_end, error_msg)) {
+        return false;
+      }
+    }
+    MemMap image_reservation;
+    MemMap oat_reservation;
+    if (!ReserveBootImageMemory(image_start,
+                                image_end,
+                                oat_end,
+                                &image_reservation,
+                                &oat_reservation,
+                                error_msg)) {
+      return false;
+    }
+
     std::vector<std::unique_ptr<ImageSpace>> spaces;
     spaces.reserve(locations.size());
     for (const std::string& location : locations) {
@@ -1534,12 +1564,12 @@
       if (!GetDalvikCacheFilename(location.c_str(), dalvik_cache_.c_str(), &filename, error_msg)) {
         return false;
       }
-      spaces.push_back(Loader::Load(location,
-                                    filename,
-                                    is_zygote_,
-                                    is_global_cache_,
-                                    validate_oat_file,
-                                    error_msg));
+      spaces.push_back(Load(location,
+                            filename,
+                            validate_oat_file,
+                            &image_reservation,
+                            &oat_reservation,
+                            error_msg));
       if (spaces.back() == nullptr) {
         return false;
       }
@@ -1560,12 +1590,56 @@
         }
       }
     }
+    if (!CheckReservationsExhausted(image_reservation, oat_reservation, error_msg)) {
+      return false;
+    }
+
     *oat_file_end = GetOatFileEnd(spaces);
     boot_image_spaces->swap(spaces);
     return true;
   }
 
  private:
+  std::unique_ptr<ImageSpace> Load(const std::string& image_location,
+                                   const std::string& image_filename,
+                                   bool validate_oat_file,
+                                   /*inout*/MemMap* image_reservation,
+                                   /*inout*/MemMap* oat_reservation,
+                                   /*out*/std::string* error_msg)
+      REQUIRES_SHARED(Locks::mutator_lock_) {
+    // Should this be a RDWR lock? This is only a defensive measure, as at
+    // this point the image should exist.
+    // However, only the zygote can write into the global dalvik-cache, so
+    // restrict to zygote processes, or any process that isn't using
+    // /data/dalvik-cache (which we assume to be allowed to write there).
+    const bool rw_lock = is_zygote_ || !is_global_cache_;
+
+    // Note that we must not use the file descriptor associated with
+    // ScopedFlock::GetFile to Init the image file. We want the file
+    // descriptor (and the associated exclusive lock) to be released when
+    // we leave Create.
+    ScopedFlock image = LockedFile::Open(image_filename.c_str(),
+                                         rw_lock ? (O_CREAT | O_RDWR) : O_RDONLY /* flags */,
+                                         true /* block */,
+                                         error_msg);
+
+    VLOG(startup) << "Using image file " << image_filename.c_str() << " for image location "
+                  << image_location;
+    // If we are in /system we can assume the image is good. We can also
+    // assume this if we are using a relocated image (i.e. image checksum
+    // matches) since this is only different by the offset. We need this to
+    // make sure that host tests continue to work.
+    // Since we are the boot image, pass null since we load the oat file from the boot image oat
+    // file name.
+    return Loader::Init(image_filename.c_str(),
+                        image_location.c_str(),
+                        validate_oat_file,
+                        /* oat_file */ nullptr,
+                        image_reservation,
+                        oat_reservation,
+                        error_msg);
+  }
+
   // Extract boot class path from oat file associated with `image_filename`
   // and list all associated image locations.
   static bool GetBootClassPathImageLocations(const std::string& image_location,
@@ -1577,10 +1651,10 @@
                                                     oat_filename,
                                                     oat_filename,
                                                     /* requested_base */ nullptr,
-                                                    /* oat_file_begin */ nullptr,
                                                     /* executable */ false,
                                                     /* low_4gb */ false,
                                                     /* abs_dex_location */ nullptr,
+                                                    /* reservation */ nullptr,
                                                     error_msg));
     if (oat_file == nullptr) {
       *error_msg = StringPrintf("Failed to open oat file '%s' for image file %s: %s",
@@ -1598,6 +1672,72 @@
     return true;
   }
 
+  bool GetBootImageAddressRange(const std::string& filename,
+                                /*out*/uint32_t* start,
+                                /*out*/uint32_t* end,
+                                /*out*/uint32_t* oat_end,
+                                /*out*/std::string* error_msg) {
+    ImageHeader system_hdr;
+    if (!ReadSpecificImageHeader(filename.c_str(), &system_hdr)) {
+      *error_msg = StringPrintf("Cannot read header of %s", filename.c_str());
+      return false;
+    }
+    *start = reinterpret_cast32<uint32_t>(system_hdr.GetImageBegin());
+    CHECK_ALIGNED(*start, kPageSize);
+    *end = RoundUp(*start + system_hdr.GetImageSize(), kPageSize);
+    *oat_end = RoundUp(reinterpret_cast32<uint32_t>(system_hdr.GetOatFileEnd()), kPageSize);
+    return true;
+  }
+
+  bool ReserveBootImageMemory(uint32_t image_start,
+                              uint32_t image_end,
+                              uint32_t oat_end,
+                              /*out*/MemMap* image_reservation,
+                              /*out*/MemMap* oat_reservation,
+                              /*out*/std::string* error_msg) {
+    DCHECK(!image_reservation->IsValid());
+    *image_reservation =
+        MemMap::MapAnonymous("Boot image reservation",
+                             reinterpret_cast32<uint8_t*>(image_start),
+                             oat_end - image_start,
+                             PROT_NONE,
+                             /* low_4gb */ true,
+                             /* reuse */ false,
+                             /* reservation */ nullptr,
+                             error_msg);
+    if (!image_reservation->IsValid()) {
+      return false;
+    }
+    DCHECK(!oat_reservation->IsValid());
+    *oat_reservation = image_reservation->RemapAtEnd(reinterpret_cast32<uint8_t*>(image_end),
+                                                     "Boot image oat reservation",
+                                                     PROT_NONE,
+                                                     error_msg);
+    if (!oat_reservation->IsValid()) {
+      return false;
+    }
+
+    return true;
+  }
+
+  bool CheckReservationsExhausted(const MemMap& image_reservation,
+                                  const MemMap& oat_reservation,
+                                  /*out*/std::string* error_msg) {
+    if (image_reservation.IsValid()) {
+      *error_msg = StringPrintf("Excessive image reservation after loading boot image: %p-%p",
+                                image_reservation.Begin(),
+                                image_reservation.End());
+      return false;
+    }
+    if (oat_reservation.IsValid()) {
+      *error_msg = StringPrintf("Excessive oat reservation after loading boot image: %p-%p",
+                                image_reservation.Begin(),
+                                image_reservation.End());
+      return false;
+    }
+    return true;
+  }
+
   uint8_t* GetOatFileEnd(const std::vector<std::unique_ptr<ImageSpace>>& spaces) {
     DCHECK(std::is_sorted(
         spaces.begin(),
@@ -1834,7 +1974,13 @@
 std::unique_ptr<ImageSpace> ImageSpace::CreateFromAppImage(const char* image,
                                                            const OatFile* oat_file,
                                                            std::string* error_msg) {
-  return Loader::Init(image, image, /*validate_oat_file*/false, oat_file, /*out*/error_msg);
+  return Loader::Init(image,
+                      image,
+                      /* validate_oat_file */ false,
+                      oat_file,
+                      /* image_reservation */ nullptr,
+                      /* oat_reservation */ nullptr,
+                      error_msg);
 }
 
 const OatFile* ImageSpace::GetOatFile() const {
diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h
index 93cf947..bd686be 100644
--- a/runtime/gc/space/image_space.h
+++ b/runtime/gc/space/image_space.h
@@ -183,7 +183,7 @@
   ImageSpace(const std::string& name,
              const char* image_location,
              MemMap&& mem_map,
-             accounting::ContinuousSpaceBitmap* live_bitmap,
+             std::unique_ptr<accounting::ContinuousSpaceBitmap> live_bitmap,
              uint8_t* end);
 
   // The OatFile associated with the image during early startup to
diff --git a/runtime/gc/space/image_space_test.cc b/runtime/gc/space/image_space_test.cc
index 347af4e..299a413 100644
--- a/runtime/gc/space/image_space_test.cc
+++ b/runtime/gc/space/image_space_test.cc
@@ -46,11 +46,11 @@
   std::unique_ptr<OatFile> oat(OatFile::Open(/* zip_fd */ -1,
                                              oat_location.c_str(),
                                              oat_location.c_str(),
-                                             nullptr,
-                                             nullptr,
-                                             false,
-                                             /*low_4gb*/false,
-                                             nullptr,
+                                             /* requested_base */ nullptr,
+                                             /* executable */ false,
+                                             /* low_4gb */ false,
+                                             /* abs_dex_location */ nullptr,
+                                             /* reservation */ nullptr,
                                              &error_msg));
   ASSERT_TRUE(oat != nullptr) << error_msg;
 
diff --git a/runtime/jit/jit_code_cache.cc b/runtime/jit/jit_code_cache.cc
index b743b40..2b2898c 100644
--- a/runtime/jit/jit_code_cache.cc
+++ b/runtime/jit/jit_code_cache.cc
@@ -207,6 +207,7 @@
       kProtData,
       /* low_4gb */ true,
       /* reuse */ false,
+      /* reservation */ nullptr,
       &error_str);
   if (!data_map.IsValid()) {
     std::ostringstream oss;
diff --git a/runtime/oat_file.cc b/runtime/oat_file.cc
index 4780aea..d5246b4 100644
--- a/runtime/oat_file.cc
+++ b/runtime/oat_file.cc
@@ -102,12 +102,12 @@
                                   const std::string& elf_filename,
                                   const std::string& location,
                                   uint8_t* requested_base,
-                                  uint8_t* oat_file_begin,
                                   bool writable,
                                   bool executable,
                                   bool low_4gb,
                                   const char* abs_dex_location,
-                                  std::string* error_msg);
+                                  /*inout*/MemMap* reservation,  // Where to load if not null.
+                                  /*out*/std::string* error_msg);
 
   template <typename kOatFileBaseSubType>
   static OatFileBase* OpenOatFile(int zip_fd,
@@ -116,12 +116,12 @@
                                   const std::string& vdex_filename,
                                   const std::string& oat_filename,
                                   uint8_t* requested_base,
-                                  uint8_t* oat_file_begin,
                                   bool writable,
                                   bool executable,
                                   bool low_4gb,
                                   const char* abs_dex_location,
-                                  std::string* error_msg);
+                                  /*inout*/MemMap* reservation,  // Where to load if not null.
+                                  /*out*/std::string* error_msg);
 
  protected:
   OatFileBase(const std::string& filename, bool executable) : OatFile(filename, executable) {}
@@ -143,18 +143,18 @@
                 std::string* error_msg);
 
   virtual bool Load(const std::string& elf_filename,
-                    uint8_t* oat_file_begin,
                     bool writable,
                     bool executable,
                     bool low_4gb,
-                    std::string* error_msg) = 0;
+                    /*inout*/MemMap* reservation,  // Where to load if not null.
+                    /*out*/std::string* error_msg) = 0;
 
   virtual bool Load(int oat_fd,
-                    uint8_t* oat_file_begin,
                     bool writable,
                     bool executable,
                     bool low_4gb,
-                    std::string* error_msg) = 0;
+                    /*inout*/MemMap* reservation,  // Where to load if not null.
+                    /*out*/std::string* error_msg) = 0;
 
   bool ComputeFields(uint8_t* requested_base,
                      const std::string& file_path,
@@ -188,21 +188,21 @@
                                       const std::string& elf_filename,
                                       const std::string& location,
                                       uint8_t* requested_base,
-                                      uint8_t* oat_file_begin,
                                       bool writable,
                                       bool executable,
                                       bool low_4gb,
                                       const char* abs_dex_location,
-                                      std::string* error_msg) {
+                                      /*inout*/MemMap* reservation,
+                                      /*out*/std::string* error_msg) {
   std::unique_ptr<OatFileBase> ret(new kOatFileBaseSubType(location, executable));
 
   ret->PreLoad();
 
   if (!ret->Load(elf_filename,
-                 oat_file_begin,
                  writable,
                  executable,
                  low_4gb,
+                 reservation,
                  error_msg)) {
     return nullptr;
   }
@@ -231,19 +231,19 @@
                                       const std::string& vdex_location,
                                       const std::string& oat_location,
                                       uint8_t* requested_base,
-                                      uint8_t* oat_file_begin,
                                       bool writable,
                                       bool executable,
                                       bool low_4gb,
                                       const char* abs_dex_location,
-                                      std::string* error_msg) {
+                                      /*inout*/MemMap* reservation,
+                                      /*out*/std::string* error_msg) {
   std::unique_ptr<OatFileBase> ret(new kOatFileBaseSubType(oat_location, executable));
 
   if (!ret->Load(oat_fd,
-                 oat_file_begin,
                  writable,
                  executable,
                  low_4gb,
+                 reservation,
                  error_msg)) {
     return nullptr;
   }
@@ -923,13 +923,18 @@
   void PreLoad() override;
 
   bool Load(const std::string& elf_filename,
-            uint8_t* oat_file_begin,
             bool writable,
             bool executable,
             bool low_4gb,
-            std::string* error_msg) override;
+            /*inout*/MemMap* reservation,  // Where to load if not null.
+            /*out*/std::string* error_msg) override;
 
-  bool Load(int, uint8_t*, bool, bool, bool, std::string*) {
+  bool Load(int oat_fd ATTRIBUTE_UNUSED,
+            bool writable ATTRIBUTE_UNUSED,
+            bool executable ATTRIBUTE_UNUSED,
+            bool low_4gb ATTRIBUTE_UNUSED,
+            /*inout*/MemMap* reservation ATTRIBUTE_UNUSED,
+            /*out*/std::string* error_msg ATTRIBUTE_UNUSED) override {
     return false;
   }
 
@@ -938,8 +943,8 @@
 
  private:
   bool Dlopen(const std::string& elf_filename,
-              uint8_t* oat_file_begin,
-              std::string* error_msg);
+              /*inout*/MemMap* reservation,  // Where to load if not null.
+              /*out*/std::string* error_msg);
 
   // On the host, if the same library is loaded again with dlopen the same
   // file handle is returned. This differs from the behavior of dlopen on the
@@ -952,12 +957,13 @@
   // Guarded by host_dlopen_handles_lock_;
   static std::unordered_set<void*> host_dlopen_handles_;
 
+  // Reservation and dummy memory map objects corresponding to the regions mapped by dlopen.
+  // Note: Must be destroyed after dlclose() as it can hold the owning reservation.
+  std::vector<MemMap> dlopen_mmaps_;
+
   // dlopen handle during runtime.
   void* dlopen_handle_;  // TODO: Unique_ptr with custom deleter.
 
-  // Dummy memory map objects corresponding to the regions mapped by dlopen.
-  std::vector<MemMap> dlopen_mmaps_;
-
   // The number of shared objects the linker told us about before loading. Used to
   // (optimistically) optimize the PreSetup stage (see comment there).
   size_t shared_objects_before_;
@@ -975,9 +981,9 @@
 #else
   // Count the entries in dl_iterate_phdr we get at this point in time.
   struct dl_iterate_context {
-    static int callback(struct dl_phdr_info *info ATTRIBUTE_UNUSED,
+    static int callback(dl_phdr_info* info ATTRIBUTE_UNUSED,
                         size_t size ATTRIBUTE_UNUSED,
-                        void *data) {
+                        void* data) {
       reinterpret_cast<dl_iterate_context*>(data)->count++;
       return 0;  // Continue iteration.
     }
@@ -990,11 +996,11 @@
 }
 
 bool DlOpenOatFile::Load(const std::string& elf_filename,
-                         uint8_t* oat_file_begin,
                          bool writable,
                          bool executable,
                          bool low_4gb,
-                         std::string* error_msg) {
+                         /*inout*/MemMap* reservation,  // Where to load if not null.
+                         /*out*/std::string* error_msg) {
   // Use dlopen only when flagged to do so, and when it's OK to load things executable.
   // TODO: Also try when not executable? The issue here could be re-mapping as writable (as
   //       !executable is a sign that we may want to patch), which may not be allowed for
@@ -1027,15 +1033,15 @@
     }
   }
 
-  bool success = Dlopen(elf_filename, oat_file_begin, error_msg);
+  bool success = Dlopen(elf_filename, reservation, error_msg);
   DCHECK(dlopen_handle_ != nullptr || !success);
 
   return success;
 }
 
 bool DlOpenOatFile::Dlopen(const std::string& elf_filename,
-                           uint8_t* oat_file_begin,
-                           std::string* error_msg) {
+                           /*inout*/MemMap* reservation,
+                           /*out*/std::string* error_msg) {
 #ifdef __APPLE__
   // The dl_iterate_phdr syscall is missing.  There is similar API on OSX,
   // but let's fallback to the custom loading code for the time being.
@@ -1056,15 +1062,85 @@
                                                                 //    times).
                     ANDROID_DLEXT_FORCE_FIXED_VADDR;            // Take a non-zero vaddr as absolute
                                                                 //   (non-pic boot image).
-    if (oat_file_begin != nullptr) {                            //
-      extinfo.flags |= ANDROID_DLEXT_LOAD_AT_FIXED_ADDRESS;     // Use the requested addr if
-      extinfo.reserved_addr = oat_file_begin;                   // vaddr = 0.
-    }                                                           //   (pic boot image).
+    if (reservation != nullptr) {
+      if (!reservation->IsValid()) {
+        *error_msg = StringPrintf("Invalid reservation for %s", elf_filename.c_str());
+        return false;
+      }
+      extinfo.flags |= ANDROID_DLEXT_RESERVED_ADDRESS;          // Use the reserved memory range.
+      extinfo.reserved_addr = reservation->Begin();
+      extinfo.reserved_size = reservation->Size();
+    }
     dlopen_handle_ = android_dlopen_ext(absolute_path.get(), RTLD_NOW, &extinfo);
+    if (reservation != nullptr && dlopen_handle_ != nullptr) {
+      // Find used pages from the reservation.
+      struct dl_iterate_context {
+        static int callback(dl_phdr_info* info, size_t size ATTRIBUTE_UNUSED, void* data) {
+          auto* context = reinterpret_cast<dl_iterate_context*>(data);
+          static_assert(std::is_same<Elf32_Half, Elf64_Half>::value, "Half must match");
+          using Elf_Half = Elf64_Half;
+
+          // See whether this callback corresponds to the file which we have just loaded.
+          uint8_t* reservation_begin = context->reservation->Begin();
+          bool contained_in_reservation = false;
+          for (Elf_Half i = 0; i < info->dlpi_phnum; i++) {
+            if (info->dlpi_phdr[i].p_type == PT_LOAD) {
+              uint8_t* vaddr = reinterpret_cast<uint8_t*>(info->dlpi_addr +
+                  info->dlpi_phdr[i].p_vaddr);
+              size_t memsz = info->dlpi_phdr[i].p_memsz;
+              size_t offset = static_cast<size_t>(vaddr - reservation_begin);
+              if (offset < context->reservation->Size()) {
+                contained_in_reservation = true;
+                DCHECK_LE(memsz, context->reservation->Size() - offset);
+              } else if (vaddr < reservation_begin) {
+                // Check that there's no overlap with the reservation.
+                DCHECK_LE(memsz, static_cast<size_t>(reservation_begin - vaddr));
+              }
+              break;  // It is sufficient to check the first PT_LOAD header.
+            }
+          }
+
+          if (contained_in_reservation) {
+            for (Elf_Half i = 0; i < info->dlpi_phnum; i++) {
+              if (info->dlpi_phdr[i].p_type == PT_LOAD) {
+                uint8_t* vaddr = reinterpret_cast<uint8_t*>(info->dlpi_addr +
+                    info->dlpi_phdr[i].p_vaddr);
+                size_t memsz = info->dlpi_phdr[i].p_memsz;
+                size_t offset = static_cast<size_t>(vaddr - reservation_begin);
+                DCHECK_LT(offset, context->reservation->Size());
+                DCHECK_LE(memsz, context->reservation->Size() - offset);
+                context->max_size = std::max(context->max_size, offset + memsz);
+              }
+            }
+
+            return 1;  // Stop iteration and return 1 from dl_iterate_phdr.
+          }
+          return 0;  // Continue iteration and return 0 from dl_iterate_phdr when finished.
+        }
+
+        const MemMap* const reservation;
+        size_t max_size = 0u;
+      };
+      dl_iterate_context context = { reservation };
+
+      if (dl_iterate_phdr(dl_iterate_context::callback, &context) == 0) {
+        LOG(FATAL) << "Could not find the shared object mmapped to the reservation.";
+        UNREACHABLE();
+      }
+
+      // Take ownership of the memory used by the shared object. dlopen() does not assume
+      // full ownership of this memory and dlclose() shall just remap it as zero pages with
+      // PROT_NONE. We need to unmap the memory when destroying this oat file.
+      dlopen_mmaps_.push_back(reservation->TakeReservedMemory(context.max_size));
+    }
 #else
-    UNUSED(oat_file_begin);
     static_assert(!kIsTargetBuild || kIsTargetLinux || kIsTargetFuchsia,
                   "host_dlopen_handles_ will leak handles");
+    if (reservation != nullptr) {
+      *error_msg = StringPrintf("dlopen() into reserved memory is unsupported on host for '%s'.",
+                                elf_filename.c_str());
+      return false;
+    }
     MutexLock mu(Thread::Current(), *Locks::host_dlopen_handles_lock_);
     dlopen_handle_ = dlopen(absolute_path.get(), RTLD_NOW);
     if (dlopen_handle_ != nullptr) {
@@ -1092,8 +1168,11 @@
   UNREACHABLE();
 #else
   struct dl_iterate_context {
-    static int callback(struct dl_phdr_info *info, size_t /* size */, void *data) {
+    static int callback(dl_phdr_info* info, size_t size ATTRIBUTE_UNUSED, void* data) {
       auto* context = reinterpret_cast<dl_iterate_context*>(data);
+      static_assert(std::is_same<Elf32_Half, Elf64_Half>::value, "Half must match");
+      using Elf_Half = Elf64_Half;
+
       context->shared_objects_seen++;
       if (context->shared_objects_seen < context->shared_objects_before) {
         // We haven't been called yet for anything we haven't seen before. Just continue.
@@ -1104,7 +1183,7 @@
 
       // See whether this callback corresponds to the file which we have just loaded.
       bool contains_begin = false;
-      for (int i = 0; i < info->dlpi_phnum; i++) {
+      for (Elf_Half i = 0; i < info->dlpi_phnum; i++) {
         if (info->dlpi_phdr[i].p_type == PT_LOAD) {
           uint8_t* vaddr = reinterpret_cast<uint8_t*>(info->dlpi_addr +
               info->dlpi_phdr[i].p_vaddr);
@@ -1117,7 +1196,7 @@
       }
       // Add dummy mmaps for this file.
       if (contains_begin) {
-        for (int i = 0; i < info->dlpi_phnum; i++) {
+        for (Elf_Half i = 0; i < info->dlpi_phnum; i++) {
           if (info->dlpi_phdr[i].p_type == PT_LOAD) {
             uint8_t* vaddr = reinterpret_cast<uint8_t*>(info->dlpi_addr +
                 info->dlpi_phdr[i].p_vaddr);
@@ -1163,13 +1242,12 @@
   static ElfOatFile* OpenElfFile(int zip_fd,
                                  File* file,
                                  const std::string& location,
-                                 uint8_t* requested_base,
-                                 uint8_t* oat_file_begin,  // Override base if not null
                                  bool writable,
                                  bool executable,
                                  bool low_4gb,
                                  const char* abs_dex_location,
-                                 std::string* error_msg);
+                                 /*inout*/MemMap* reservation,  // Where to load if not null.
+                                 /*out*/std::string* error_msg);
 
   bool InitializeFromElfFile(int zip_fd,
                              ElfFile* elf_file,
@@ -1191,29 +1269,29 @@
   }
 
   bool Load(const std::string& elf_filename,
-            uint8_t* oat_file_begin,  // Override where the file is loaded to if not null
             bool writable,
             bool executable,
             bool low_4gb,
-            std::string* error_msg) override;
+            /*inout*/MemMap* reservation,  // Where to load if not null.
+            /*out*/std::string* error_msg) override;
 
   bool Load(int oat_fd,
-            uint8_t* oat_file_begin,  // Override where the file is loaded to if not null
             bool writable,
             bool executable,
             bool low_4gb,
-            std::string* error_msg) override;
+            /*inout*/MemMap* reservation,  // Where to load if not null.
+            /*out*/std::string* error_msg) override;
 
   void PreSetup(const std::string& elf_filename ATTRIBUTE_UNUSED) override {
   }
 
  private:
   bool ElfFileOpen(File* file,
-                   uint8_t* oat_file_begin,  // Override where the file is loaded to if not null
                    bool writable,
                    bool executable,
                    bool low_4gb,
-                   std::string* error_msg);
+                   /*inout*/MemMap* reservation,  // Where to load if not null.
+                   /*out*/std::string* error_msg);
 
  private:
   // Backing memory map for oat file during cross compilation.
@@ -1225,20 +1303,19 @@
 ElfOatFile* ElfOatFile::OpenElfFile(int zip_fd,
                                     File* file,
                                     const std::string& location,
-                                    uint8_t* requested_base,
-                                    uint8_t* oat_file_begin,  // Override base if not null
                                     bool writable,
                                     bool executable,
                                     bool low_4gb,
                                     const char* abs_dex_location,
-                                    std::string* error_msg) {
+                                    /*inout*/MemMap* reservation,  // Where to load if not null.
+                                    /*out*/std::string* error_msg) {
   ScopedTrace trace("Open elf file " + location);
   std::unique_ptr<ElfOatFile> oat_file(new ElfOatFile(location, executable));
   bool success = oat_file->ElfFileOpen(file,
-                                       oat_file_begin,
                                        writable,
                                        low_4gb,
                                        executable,
+                                       reservation,
                                        error_msg);
   if (!success) {
     CHECK(!error_msg->empty());
@@ -1246,7 +1323,7 @@
   }
 
   // Complete the setup.
-  if (!oat_file->ComputeFields(requested_base, file->GetPath(), error_msg)) {
+  if (!oat_file->ComputeFields(/* requested_base */ nullptr, file->GetPath(), error_msg)) {
     return nullptr;
   }
 
@@ -1279,11 +1356,11 @@
 }
 
 bool ElfOatFile::Load(const std::string& elf_filename,
-                      uint8_t* oat_file_begin,  // Override where the file is loaded to if not null
                       bool writable,
                       bool executable,
                       bool low_4gb,
-                      std::string* error_msg) {
+                      /*inout*/MemMap* reservation,
+                      /*out*/std::string* error_msg) {
   ScopedTrace trace(__PRETTY_FUNCTION__);
   std::unique_ptr<File> file(OS::OpenFileForReading(elf_filename.c_str()));
   if (file == nullptr) {
@@ -1291,19 +1368,19 @@
     return false;
   }
   return ElfOatFile::ElfFileOpen(file.get(),
-                                 oat_file_begin,
                                  writable,
                                  executable,
                                  low_4gb,
+                                 reservation,
                                  error_msg);
 }
 
 bool ElfOatFile::Load(int oat_fd,
-                      uint8_t* oat_file_begin,  // Override where the file is loaded to if not null
                       bool writable,
                       bool executable,
                       bool low_4gb,
-                      std::string* error_msg) {
+                      /*inout*/MemMap* reservation,
+                      /*out*/std::string* error_msg) {
   ScopedTrace trace(__PRETTY_FUNCTION__);
   if (oat_fd != -1) {
     std::unique_ptr<File> file = std::make_unique<File>(oat_fd, false);
@@ -1314,34 +1391,33 @@
       return false;
     }
     return ElfOatFile::ElfFileOpen(file.get(),
-                                   oat_file_begin,
                                    writable,
                                    executable,
                                    low_4gb,
+                                   reservation,
                                    error_msg);
   }
   return false;
 }
 
 bool ElfOatFile::ElfFileOpen(File* file,
-                             uint8_t* oat_file_begin,
                              bool writable,
                              bool executable,
                              bool low_4gb,
-                             std::string* error_msg) {
+                             /*inout*/MemMap* reservation,
+                             /*out*/std::string* error_msg) {
   ScopedTrace trace(__PRETTY_FUNCTION__);
   // TODO: rename requested_base to oat_data_begin
   elf_file_.reset(ElfFile::Open(file,
                                 writable,
                                 /*program_header_only*/true,
                                 low_4gb,
-                                error_msg,
-                                oat_file_begin));
+                                error_msg));
   if (elf_file_ == nullptr) {
     DCHECK(!error_msg->empty());
     return false;
   }
-  bool loaded = elf_file_->Load(file, executable, low_4gb, error_msg);
+  bool loaded = elf_file_->Load(file, executable, low_4gb, reservation, error_msg);
   DCHECK(loaded || !error_msg->empty());
   return loaded;
 }
@@ -1392,11 +1468,11 @@
                        const std::string& oat_filename,
                        const std::string& oat_location,
                        uint8_t* requested_base,
-                       uint8_t* oat_file_begin,
                        bool executable,
                        bool low_4gb,
                        const char* abs_dex_location,
-                       std::string* error_msg) {
+                       /*inout*/MemMap* reservation,
+                       /*out*/std::string* error_msg) {
   ScopedTrace trace("Open oat file " + oat_location);
   CHECK(!oat_filename.empty()) << oat_location;
   CheckLocation(oat_location);
@@ -1419,11 +1495,11 @@
                                                                  oat_filename,
                                                                  oat_location,
                                                                  requested_base,
-                                                                 oat_file_begin,
                                                                  false /* writable */,
                                                                  executable,
                                                                  low_4gb,
                                                                  abs_dex_location,
+                                                                 reservation,
                                                                  error_msg);
   if (with_dlopen != nullptr) {
     return with_dlopen;
@@ -1449,11 +1525,11 @@
                                                                 oat_filename,
                                                                 oat_location,
                                                                 requested_base,
-                                                                oat_file_begin,
                                                                 false /* writable */,
                                                                 executable,
                                                                 low_4gb,
                                                                 abs_dex_location,
+                                                                reservation,
                                                                 error_msg);
   return with_internal;
 }
@@ -1463,11 +1539,11 @@
                        int oat_fd,
                        const std::string& oat_location,
                        uint8_t* requested_base,
-                       uint8_t* oat_file_begin,
                        bool executable,
                        bool low_4gb,
                        const char* abs_dex_location,
-                       std::string* error_msg) {
+                       /*inout*/MemMap* reservation,
+                       /*out*/std::string* error_msg) {
   CHECK(!oat_location.empty()) << oat_location;
 
   std::string vdex_location = GetVdexFilename(oat_location);
@@ -1478,11 +1554,11 @@
                                                                 vdex_location,
                                                                 oat_location,
                                                                 requested_base,
-                                                                oat_file_begin,
                                                                 false /* writable */,
                                                                 executable,
                                                                 low_4gb,
                                                                 abs_dex_location,
+                                                                reservation,
                                                                 error_msg);
   return with_internal;
 }
@@ -1496,12 +1572,11 @@
   return ElfOatFile::OpenElfFile(zip_fd,
                                  file,
                                  location,
-                                 nullptr,
-                                 nullptr,
-                                 true,
-                                 false,
+                                 /* writable */ true,
+                                 /* executable */ false,
                                  /*low_4gb*/false,
                                  abs_dex_location,
+                                 /* reservation */ nullptr,
                                  error_msg);
 }
 
@@ -1514,12 +1589,11 @@
   return ElfOatFile::OpenElfFile(zip_fd,
                                  file,
                                  location,
-                                 nullptr,
-                                 nullptr,
-                                 false,
-                                 false,
+                                 /* writable */ false,
+                                 /* executable */ false,
                                  /*low_4gb*/false,
                                  abs_dex_location,
+                                 /* reservation */ nullptr,
                                  error_msg);
 }
 
diff --git a/runtime/oat_file.h b/runtime/oat_file.h
index 21e2144..f20c603 100644
--- a/runtime/oat_file.h
+++ b/runtime/oat_file.h
@@ -86,11 +86,11 @@
                        const std::string& filename,
                        const std::string& location,
                        uint8_t* requested_base,
-                       uint8_t* oat_file_begin,
                        bool executable,
                        bool low_4gb,
                        const char* abs_dex_location,
-                       std::string* error_msg);
+                       /*inout*/MemMap* reservation,  // Where to load if not null.
+                       /*out*/std::string* error_msg);
 
   // Similar to OatFile::Open(const std::string...), but accepts input vdex and
   // odex files as file descriptors. We also take zip_fd in case the vdex does not
@@ -100,11 +100,11 @@
                        int oat_fd,
                        const std::string& oat_location,
                        uint8_t* requested_base,
-                       uint8_t* oat_file_begin,
                        bool executable,
                        bool low_4gb,
                        const char* abs_dex_location,
-                       std::string* error_msg);
+                       /*inout*/MemMap* reservation,  // Where to load if not null.
+                       /*out*/std::string* error_msg);
 
   // Open an oat file from an already opened File.
   // Does not use dlopen underneath so cannot be used for runtime use
diff --git a/runtime/oat_file_assistant.cc b/runtime/oat_file_assistant.cc
index e262ff7..4ed7e35 100644
--- a/runtime/oat_file_assistant.cc
+++ b/runtime/oat_file_assistant.cc
@@ -831,22 +831,22 @@
                                     vdex_fd_,
                                     oat_fd_,
                                     filename_.c_str(),
-                                    nullptr,
-                                    nullptr,
+                                    /* requested_base */ nullptr,
                                     executable,
-                                    false /* low_4gb */,
+                                    /* low_4gb */ false,
                                     oat_file_assistant_->dex_location_.c_str(),
+                                    /* reservation */ nullptr,
                                     &error_msg));
         }
       } else {
         file_.reset(OatFile::Open(/* zip_fd */ -1,
                                   filename_.c_str(),
                                   filename_.c_str(),
-                                  nullptr,
-                                  nullptr,
+                                  /* requested_base */ nullptr,
                                   executable,
-                                  false /* low_4gb */,
+                                  /* low_4gb */ false,
                                   oat_file_assistant_->dex_location_.c_str(),
+                                  /* reservation */ nullptr,
                                   &error_msg));
       }
       if (file_.get() == nullptr) {
diff --git a/runtime/oat_file_test.cc b/runtime/oat_file_test.cc
index 12dfe20..51d8fca 100644
--- a/runtime/oat_file_test.cc
+++ b/runtime/oat_file_test.cc
@@ -77,11 +77,11 @@
   std::unique_ptr<OatFile> odex_file(OatFile::Open(/* zip_fd */ -1,
                                                    oat_location.c_str(),
                                                    oat_location.c_str(),
-                                                   nullptr,
-                                                   nullptr,
-                                                   false,
-                                                   /*low_4gb*/false,
+                                                   /* requested_base */ nullptr,
+                                                   /* executable */ false,
+                                                   /* low_4gb */ false,
                                                    dex_location.c_str(),
+                                                   /* reservation */ nullptr,
                                                    &error_msg));
   ASSERT_TRUE(odex_file.get() != nullptr);
 
@@ -105,11 +105,11 @@
     std::unique_ptr<OatFile> odex_file(OatFile::Open(/* zip_fd */ -1,
                                                      oat_location.c_str(),
                                                      oat_location.c_str(),
-                                                     nullptr,
-                                                     nullptr,
-                                                     false,
-                                                     /*low_4gb*/false,
+                                                     /* requested_base */ nullptr,
+                                                     /* executable */ false,
+                                                     /* low_4gb */ false,
                                                      dex_location.c_str(),
+                                                     /* reservation */ nullptr,
                                                      &error_msg));
     ASSERT_TRUE(odex_file != nullptr);
     ASSERT_EQ(2u, odex_file->GetOatDexFiles().size());
@@ -120,13 +120,13 @@
 
   // And try to load again.
   std::unique_ptr<OatFile> odex_file(OatFile::Open(/* zip_fd */ -1,
-                                                   oat_location.c_str(),
-                                                   oat_location.c_str(),
-                                                   nullptr,
-                                                   nullptr,
-                                                   false,
-                                                   /*low_4gb*/false,
+                                                   oat_location,
+                                                   oat_location,
+                                                   /* requested_base */ nullptr,
+                                                   /* executable */ false,
+                                                   /* low_4gb */ false,
                                                    dex_location.c_str(),
+                                                   /* reservation */ nullptr,
                                                    &error_msg));
   EXPECT_TRUE(odex_file == nullptr);
   EXPECT_NE(std::string::npos, error_msg.find("expected 2 uncompressed dex files, but found 1"))
diff --git a/runtime/vdex_file.cc b/runtime/vdex_file.cc
index a7dc225..452cd8e 100644
--- a/runtime/vdex_file.cc
+++ b/runtime/vdex_file.cc
@@ -150,10 +150,11 @@
       (writable || unquicken) ? PROT_READ | PROT_WRITE : PROT_READ,
       unquicken ? MAP_PRIVATE : MAP_SHARED,
       file_fd,
-      0 /* start offset */,
+      /* start */ 0u,
       low_4gb,
-      mmap_reuse,
       vdex_filename.c_str(),
+      mmap_reuse,
+      /* reservation */ nullptr,
       error_msg);
   if (!mmap.IsValid()) {
     *error_msg = "Failed to mmap file " + vdex_filename + " : " + *error_msg;