Merge tag 'android-security-11.0.0_r49' into int/11/fp3

Android security 11.0.0 release 49

* tag 'android-security-11.0.0_r49':
  Only restore apex backups on non-checkpointing devices.

Change-Id: I1a9839731b5f385ea4bb2967b180615e78315afc
diff --git a/apexd/Android.bp b/apexd/Android.bp
index e8d221d..b2f8202 100644
--- a/apexd/Android.bp
+++ b/apexd/Android.bp
@@ -291,7 +291,9 @@
     "apex_database_test.cpp",
     "apex_file_test.cpp",
     "apex_manifest_test.cpp",
+    "apexd_session_test.cpp",
     "apexd_verity_test.cpp",
+    "apexd_utils_test.cpp",
     "apexservice_test.cpp",
   ],
   host_supported: false,
diff --git a/apexd/apexd.cpp b/apexd/apexd.cpp
index 80e075a..aae1005 100644
--- a/apexd/apexd.cpp
+++ b/apexd/apexd.cpp
@@ -1321,30 +1321,7 @@
 //  Migrates sessions directory from /data/apex/sessions to
 //  /metadata/apex/sessions, if necessary.
 Result<void> migrateSessionsDirIfNeeded() {
-  namespace fs = std::filesystem;
-  auto from_path = std::string(kApexDataDir) + "/sessions";
-  auto exists = PathExists(from_path);
-  if (!exists) {
-    return Error() << "Failed to access " << from_path << ": "
-                   << exists.error();
-  }
-  if (!*exists) {
-    LOG(DEBUG) << from_path << " does not exist. Nothing to migrate.";
-    return {};
-  }
-  auto to_path = kApexSessionsDir;
-  std::error_code error_code;
-  fs::copy(from_path, to_path, fs::copy_options::recursive, error_code);
-  if (error_code) {
-    return Error() << "Failed to copy old sessions directory"
-                   << error_code.message();
-  }
-  fs::remove_all(from_path, error_code);
-  if (error_code) {
-    return Error() << "Failed to delete old sessions directory "
-                   << error_code.message();
-  }
-  return {};
+  return ApexSession::MigrateToMetadataSessionsDir();
 }
 
 Result<void> destroySnapshots(const std::string& base_dir,
@@ -1456,7 +1433,7 @@
 }
 
 void scanStagedSessionsDirAndStage() {
-  LOG(INFO) << "Scanning " << kApexSessionsDir
+  LOG(INFO) << "Scanning " << ApexSession::GetSessionsDir()
             << " looking for sessions to be activated.";
 
   auto sessionsToActivate =
diff --git a/apexd/apexd_loop.cpp b/apexd/apexd_loop.cpp
index 568eb05..5ca096b 100644
--- a/apexd/apexd_loop.cpp
+++ b/apexd/apexd_loop.cpp
@@ -18,6 +18,8 @@
 
 #include "apexd_loop.h"
 
+#include <mutex>
+
 #include <dirent.h>
 #include <fcntl.h>
 #include <linux/fs.h>
@@ -41,6 +43,17 @@
 using android::base::StringPrintf;
 using android::base::unique_fd;
 
+#ifndef LOOP_CONFIGURE
+// These can be removed whenever we pull in the Linux v5.8 UAPI headers
+struct loop_config {
+  __u32 fd;
+  __u32 block_size;
+  struct loop_info64 info;
+  __u64 __reserved[8];
+};
+#define LOOP_CONFIGURE 0x4C0A
+#endif
+
 namespace android {
 namespace apex {
 namespace loop {
@@ -125,6 +138,99 @@
   return {};
 }
 
+Result<void> configureLoopDevice(const int device_fd, const std::string& target,
+                                 const int32_t imageOffset,
+                                 const size_t imageSize) {
+  static bool useLoopConfigure;
+  static std::once_flag onceFlag;
+  std::call_once(onceFlag, [&]() {
+    // LOOP_CONFIGURE is a new ioctl in Linux 5.8 (and backported in Android
+    // common) that allows atomically configuring a loop device. It is a lot
+    // faster than the traditional LOOP_SET_FD/LOOP_SET_STATUS64 combo, but
+    // it may not be available on updating devices, so try once before
+    // deciding.
+    struct loop_config config;
+    memset(&config, 0, sizeof(config));
+    config.fd = -1;
+    if (ioctl(device_fd, LOOP_CONFIGURE, &config) == -1 && errno == EBADF) {
+      // If the IOCTL exists, it will fail with EBADF for the -1 fd
+      useLoopConfigure = true;
+    }
+  });
+
+  /*
+   * Using O_DIRECT will tell the kernel that we want to use Direct I/O
+   * on the underlying file, which we want to do to avoid double caching.
+   * Note that Direct I/O won't be enabled immediately, because the block
+   * size of the underlying block device may not match the default loop
+   * device block size (512); when we call LOOP_SET_BLOCK_SIZE below, the
+   * kernel driver will automatically enable Direct I/O when it sees that
+   * condition is now met.
+   */
+  unique_fd target_fd(open(target.c_str(), O_RDONLY | O_CLOEXEC | O_DIRECT));
+  if (target_fd.get() == -1) {
+    return ErrnoError() << "Failed to open " << target;
+  }
+
+  struct loop_info64 li;
+  memset(&li, 0, sizeof(li));
+  strlcpy((char*)li.lo_crypt_name, kApexLoopIdPrefix, LO_NAME_SIZE);
+  li.lo_offset = imageOffset;
+  li.lo_sizelimit = imageSize;
+
+  if (useLoopConfigure) {
+    struct loop_config config;
+    memset(&config, 0, sizeof(config));
+    li.lo_flags |= LO_FLAGS_DIRECT_IO;
+    config.fd = target_fd.get();
+    config.info = li;
+    config.block_size = 4096;
+
+    if (ioctl(device_fd, LOOP_CONFIGURE, &config) == -1) {
+      return ErrnoError() << "Failed to LOOP_CONFIGURE";
+    }
+
+    return {};
+  } else {
+    if (ioctl(device_fd, LOOP_SET_FD, target_fd.get()) == -1) {
+      return ErrnoError() << "Failed to LOOP_SET_FD";
+    }
+
+    if (ioctl(device_fd, LOOP_SET_STATUS64, &li) == -1) {
+      return ErrnoError() << "Failed to LOOP_SET_STATUS64";
+    }
+
+    if (ioctl(device_fd, BLKFLSBUF, 0) == -1) {
+      // This works around a kernel bug where the following happens.
+      // 1) The device runs with a value of loop.max_part > 0
+      // 2) As part of LOOP_SET_FD above, we do a partition scan, which loads
+      //    the first 2 pages of the underlying file into the buffer cache
+      // 3) When we then change the offset with LOOP_SET_STATUS64, those pages
+      //    are not invalidated from the cache.
+      // 4) When we try to mount an ext4 filesystem on the loop device, the ext4
+      //    code will try to find a superblock by reading 4k at offset 0; but,
+      //    because we still have the old pages at offset 0 lying in the cache,
+      //    those pages will be returned directly. However, those pages contain
+      //    the data at offset 0 in the underlying file, not at the offset that
+      //    we configured
+      // 5) the ext4 driver fails to find a superblock in the (wrong) data, and
+      //    fails to mount the filesystem.
+      //
+      // To work around this, explicitly flush the block device, which will
+      // flush the buffer cache and make sure we actually read the data at the
+      // correct offset.
+      return ErrnoError() << "Failed to flush buffers on the loop device";
+    }
+
+    // Direct-IO requires the loop device to have the same block size as the
+    // underlying filesystem.
+    if (ioctl(device_fd, LOOP_SET_BLOCK_SIZE, 4096) == -1) {
+      PLOG(WARNING) << "Failed to LOOP_SET_BLOCK_SIZE";
+    }
+  }
+  return {};
+}
+
 Result<LoopbackDeviceUniqueFd> createLoopDevice(const std::string& target,
                                                 const int32_t imageOffset,
                                                 const size_t imageSize) {
@@ -140,19 +246,6 @@
 
   std::string device = StringPrintf("/dev/block/loop%d", num);
 
-  /*
-   * Using O_DIRECT will tell the kernel that we want to use Direct I/O
-   * on the underlying file, which we want to do to avoid double caching.
-   * Note that Direct I/O won't be enabled immediately, because the block
-   * size of the underlying block device may not match the default loop
-   * device block size (512); when we call LOOP_SET_BLOCK_SIZE below, the
-   * kernel driver will automatically enable Direct I/O when it sees that
-   * condition is now met.
-   */
-  unique_fd target_fd(open(target.c_str(), O_RDONLY | O_CLOEXEC | O_DIRECT));
-  if (target_fd.get() == -1) {
-    return ErrnoError() << "Failed to open " << target;
-  }
   LoopbackDeviceUniqueFd device_fd;
   {
     // See comment on kLoopDeviceRetryAttempts.
@@ -173,45 +266,10 @@
     CHECK_NE(device_fd.get(), -1);
   }
 
-  if (ioctl(device_fd.get(), LOOP_SET_FD, target_fd.get()) == -1) {
-    return ErrnoError() << "Failed to LOOP_SET_FD";
-  }
-
-  struct loop_info64 li;
-  memset(&li, 0, sizeof(li));
-  strlcpy((char*)li.lo_crypt_name, kApexLoopIdPrefix, LO_NAME_SIZE);
-  li.lo_offset = imageOffset;
-  li.lo_sizelimit = imageSize;
-  if (ioctl(device_fd.get(), LOOP_SET_STATUS64, &li) == -1) {
-    return ErrnoError() << "Failed to LOOP_SET_STATUS64";
-  }
-
-  if (ioctl(device_fd.get(), BLKFLSBUF, 0) == -1) {
-    // This works around a kernel bug where the following happens.
-    // 1) The device runs with a value of loop.max_part > 0
-    // 2) As part of LOOP_SET_FD above, we do a partition scan, which loads
-    //    the first 2 pages of the underlying file into the buffer cache
-    // 3) When we then change the offset with LOOP_SET_STATUS64, those pages
-    //    are not invalidated from the cache.
-    // 4) When we try to mount an ext4 filesystem on the loop device, the ext4
-    //    code will try to find a superblock by reading 4k at offset 0; but,
-    //    because we still have the old pages at offset 0 lying in the cache,
-    //    those pages will be returned directly. However, those pages contain
-    //    the data at offset 0 in the underlying file, not at the offset that
-    //    we configured
-    // 5) the ext4 driver fails to find a superblock in the (wrong) data, and
-    //    fails to mount the filesystem.
-    //
-    // To work around this, explicitly flush the block device, which will flush
-    // the buffer cache and make sure we actually read the data at the correct
-    // offset.
-    return ErrnoError() << "Failed to flush buffers on the loop device";
-  }
-
-  // Direct-IO requires the loop device to have the same block size as the
-  // underlying filesystem.
-  if (ioctl(device_fd.get(), LOOP_SET_BLOCK_SIZE, 4096) == -1) {
-    PLOG(WARNING) << "Failed to LOOP_SET_BLOCK_SIZE";
+  Result<void> configureStatus =
+      configureLoopDevice(device_fd.get(), target, imageOffset, imageSize);
+  if (!configureStatus.ok()) {
+    return configureStatus.error();
   }
 
   Result<void> readAheadStatus = configureReadAhead(device);
diff --git a/apexd/apexd_main.cpp b/apexd/apexd_main.cpp
index c60f4fa..d460150 100644
--- a/apexd/apexd_main.cpp
+++ b/apexd/apexd_main.cpp
@@ -133,7 +133,10 @@
 
   bool booting = android::apex::isBooting();
   if (booting) {
-    android::apex::migrateSessionsDirIfNeeded();
+    if (auto res = android::apex::migrateSessionsDirIfNeeded(); !res.ok()) {
+      LOG(ERROR) << "Failed to migrate sessions to /metadata partition : "
+                 << res.error();
+    }
     android::apex::onStart();
   }
   android::apex::binder::CreateAndRegisterService();
diff --git a/apexd/apexd_session.cpp b/apexd/apexd_session.cpp
index 3d916c4..b8dd87b 100644
--- a/apexd/apexd_session.cpp
+++ b/apexd/apexd_session.cpp
@@ -22,6 +22,7 @@
 #include "session_state.pb.h"
 
 #include <android-base/logging.h>
+#include <android-base/stringprintf.h>
 #include <dirent.h>
 #include <sys/stat.h>
 
@@ -32,6 +33,7 @@
 
 using android::base::Error;
 using android::base::Result;
+using android::base::StringPrintf;
 using apex::proto::SessionState;
 
 namespace android {
@@ -39,60 +41,49 @@
 
 namespace {
 
+// Starting from R, apexd prefers /metadata partition (kNewApexSessionsDir) as
+// location for sessions-related information. For devices that don't have
+// /metadata partition, apexd will fallback to the /data one
+// (kOldApexSessionsDir).
+static constexpr const char* kOldApexSessionsDir = "/data/apex/sessions";
+static constexpr const char* kNewApexSessionsDir = "/metadata/apex/sessions";
+
 static constexpr const char* kStateFileName = "state";
 
-std::string getSessionDir(int session_id) {
-  return kApexSessionsDir + "/" + std::to_string(session_id);
-}
-
-std::string getSessionStateFilePath(int session_id) {
-  return getSessionDir(session_id) + "/" + kStateFileName;
-}
-
-Result<std::string> createSessionDirIfNeeded(int session_id) {
-  // create /data/sessions
-  auto res = createDirIfNeeded(kApexSessionsDir, 0700);
-  if (!res.ok()) {
-    return res.error();
-  }
-  // create /data/sessions/session_id
-  std::string sessionDir = getSessionDir(session_id);
-  res = createDirIfNeeded(sessionDir, 0700);
-  if (!res.ok()) {
-    return res.error();
-  }
-
-  return sessionDir;
-}
-
-Result<void> deleteSessionDir(int session_id) {
-  std::string session_dir = getSessionDir(session_id);
-  LOG(DEBUG) << "Deleting " << session_dir;
-  auto path = std::filesystem::path(session_dir);
-  std::error_code error_code;
-  std::filesystem::remove_all(path, error_code);
-  if (error_code) {
-    return Error() << "Failed to delete " << session_dir << " : "
-                   << error_code.message();
-  }
-  return {};
-}
-
 }  // namespace
 
 ApexSession::ApexSession(SessionState state) : state_(std::move(state)) {}
 
+std::string ApexSession::GetSessionsDir() {
+  static std::string result;
+  static std::once_flag once_flag;
+  std::call_once(once_flag, [&]() {
+    auto status =
+        FindFirstExistingDirectory(kNewApexSessionsDir, kOldApexSessionsDir);
+    if (!status.ok()) {
+      LOG(FATAL) << status.error();
+    }
+    result = std::move(*status);
+  });
+  return result;
+}
+
+Result<void> ApexSession::MigrateToMetadataSessionsDir() {
+  return MoveDir(kOldApexSessionsDir, kNewApexSessionsDir);
+}
+
 Result<ApexSession> ApexSession::CreateSession(int session_id) {
   SessionState state;
   // Create session directory
-  auto sessionPath = createSessionDirIfNeeded(session_id);
-  if (!sessionPath.ok()) {
-    return sessionPath.error();
+  std::string session_dir = GetSessionsDir() + "/" + std::to_string(session_id);
+  if (auto status = createDirIfNeeded(session_dir, 0700); !status.ok()) {
+    return status.error();
   }
   state.set_id(session_id);
 
   return ApexSession(state);
 }
+
 Result<ApexSession> ApexSession::GetSessionFromFile(const std::string& path) {
   SessionState state;
   std::fstream stateFile(path, std::ios::in | std::ios::binary);
@@ -108,7 +99,8 @@
 }
 
 Result<ApexSession> ApexSession::GetSession(int session_id) {
-  auto path = getSessionStateFilePath(session_id);
+  auto path = StringPrintf("%s/%d/%s", GetSessionsDir().c_str(), session_id,
+                           kStateFileName);
 
   return GetSessionFromFile(path);
 }
@@ -117,7 +109,7 @@
   std::vector<ApexSession> sessions;
 
   Result<std::vector<std::string>> sessionPaths = ReadDir(
-      kApexSessionsDir, [](const std::filesystem::directory_entry& entry) {
+      GetSessionsDir(), [](const std::filesystem::directory_entry& entry) {
         std::error_code ec;
         return entry.is_directory(ec);
       });
@@ -238,19 +230,29 @@
     const SessionState::State& session_state) {
   state_.set_state(session_state);
 
-  auto stateFilePath = getSessionStateFilePath(state_.id());
+  auto state_file_path = StringPrintf("%s/%d/%s", GetSessionsDir().c_str(),
+                                      state_.id(), kStateFileName);
 
-  std::fstream stateFile(stateFilePath,
-                         std::ios::out | std::ios::trunc | std::ios::binary);
-  if (!state_.SerializeToOstream(&stateFile)) {
-    return Error() << "Failed to write state file " << stateFilePath;
+  std::fstream state_file(state_file_path,
+                          std::ios::out | std::ios::trunc | std::ios::binary);
+  if (!state_.SerializeToOstream(&state_file)) {
+    return Error() << "Failed to write state file " << state_file_path;
   }
 
   return {};
 }
 
 Result<void> ApexSession::DeleteSession() const {
-  return deleteSessionDir(GetId());
+  std::string session_dir = GetSessionsDir() + "/" + std::to_string(GetId());
+  LOG(INFO) << "Deleting " << session_dir;
+  auto path = std::filesystem::path(session_dir);
+  std::error_code error_code;
+  std::filesystem::remove_all(path, error_code);
+  if (error_code) {
+    return Error() << "Failed to delete " << session_dir << " : "
+                   << error_code.message();
+  }
+  return {};
 }
 
 std::ostream& operator<<(std::ostream& out, const ApexSession& session) {
diff --git a/apexd/apexd_session.h b/apexd/apexd_session.h
index e0cea91..fa977ef 100644
--- a/apexd/apexd_session.h
+++ b/apexd/apexd_session.h
@@ -28,10 +28,18 @@
 namespace android {
 namespace apex {
 
-static const std::string kApexSessionsDir = "/metadata/apex/sessions";
-
 class ApexSession {
  public:
+  // Returns top-level directory to store sessions metadata in.
+  // If device has /metadata partition, this will return
+  // /metadata/apex/sessions, on all other devices it will return
+  // /data/apex/sessions.
+  static std::string GetSessionsDir();
+  // Migrates content of /data/apex/sessions to /metadata/apex/sessions.
+  // If device doesn't have /metadata partition this call will be a no-op.
+  // If /data/apex/sessions this call will also be a no-op.
+  static android::base::Result<void> MigrateToMetadataSessionsDir();
+
   static android::base::Result<ApexSession> CreateSession(int session_id);
   static android::base::Result<ApexSession> GetSession(int session_id);
   static std::vector<ApexSession> GetSessions();
diff --git a/apexd/apexd_session_test.cpp b/apexd/apexd_session_test.cpp
new file mode 100644
index 0000000..0dc3a78
--- /dev/null
+++ b/apexd/apexd_session_test.cpp
@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+#include <errno.h>
+
+#include <android-base/file.h>
+#include <android-base/result.h>
+#include <android-base/scopeguard.h>
+#include <android-base/stringprintf.h>
+#include <android-base/strings.h>
+#include <gtest/gtest.h>
+
+#include "apexd_session.h"
+#include "apexd_test_utils.h"
+#include "apexd_utils.h"
+#include "session_state.pb.h"
+
+namespace android {
+namespace apex {
+namespace {
+
+using android::apex::testing::IsOk;
+using android::base::Join;
+using android::base::make_scope_guard;
+
+// TODO(b/170329726): add unit tests for apexd_sessions.h
+
+TEST(ApexdSessionTest, GetSessionsDirSessionsStoredInMetadata) {
+  if (access("/metadata", F_OK) != 0) {
+    GTEST_SKIP() << "Device doesn't have /metadata partition";
+  }
+
+  std::string result = ApexSession::GetSessionsDir();
+  ASSERT_EQ(result, "/metadata/apex/sessions");
+}
+
+TEST(ApexdSessionTest, GetSessionsDirNoMetadataPartitionFallbackToData) {
+  if (access("/metadata", F_OK) == 0) {
+    GTEST_SKIP() << "Device has /metadata partition";
+  }
+
+  std::string result = ApexSession::GetSessionsDir();
+  ASSERT_EQ(result, "/data/apex/sessions");
+}
+
+TEST(ApexdSessionTest, MigrateToMetadataSessionsDir) {
+  namespace fs = std::filesystem;
+
+  if (access("/metadata", F_OK) != 0) {
+    GTEST_SKIP() << "Device doesn't have /metadata partition";
+  }
+
+  // This is ugly, but does the job. To have a truly hermetic unit tests we
+  // need to refactor ApexSession class.
+  for (const auto& entry : fs::directory_iterator("/metadata/apex/sessions")) {
+    fs::remove_all(entry.path());
+  }
+
+  // This is a very ugly test set up, but to have something better we need to
+  // refactor ApexSession class.
+  class TestApexSession {
+   public:
+    TestApexSession(int id, const SessionState::State& state) {
+      path_ = "/data/apex/sessions/" + std::to_string(id);
+      if (auto status = createDirIfNeeded(path_, 0700); !status.ok()) {
+        ADD_FAILURE() << "Failed to create " << path_ << " : "
+                      << status.error();
+      }
+      SessionState session;
+      session.set_id(id);
+      session.set_state(state);
+      std::fstream state_file(
+          path_ + "/state", std::ios::out | std::ios::trunc | std::ios::binary);
+      if (!session.SerializeToOstream(&state_file)) {
+        ADD_FAILURE() << "Failed to write to " << path_;
+      }
+    }
+
+    ~TestApexSession() { fs::remove_all(path_); }
+
+   private:
+    std::string path_;
+  };
+
+  auto deleter = make_scope_guard([&]() {
+    fs::remove_all("/metadata/apex/sessions/239");
+    fs::remove_all("/metadata/apex/sessions/1543");
+  });
+
+  TestApexSession session1(239, SessionState::SUCCESS);
+  TestApexSession session2(1543, SessionState::ACTIVATION_FAILED);
+
+  ASSERT_TRUE(IsOk(ApexSession::MigrateToMetadataSessionsDir()));
+
+  auto sessions = ApexSession::GetSessions();
+  ASSERT_EQ(2u, sessions.size()) << Join(sessions, ',');
+
+  auto migrated_session_1 = ApexSession::GetSession(239);
+  ASSERT_TRUE(IsOk(migrated_session_1));
+  ASSERT_EQ(SessionState::SUCCESS, migrated_session_1->GetState());
+
+  auto migrated_session_2 = ApexSession::GetSession(1543);
+  ASSERT_TRUE(IsOk(migrated_session_2));
+  ASSERT_EQ(SessionState::ACTIVATION_FAILED, migrated_session_2->GetState());
+}
+
+}  // namespace
+}  // namespace apex
+}  // namespace android
diff --git a/apexd/apexd_utils.h b/apexd/apexd_utils.h
index 2759f9a..6997efc 100644
--- a/apexd/apexd_utils.h
+++ b/apexd/apexd_utils.h
@@ -21,6 +21,7 @@
 #include <filesystem>
 #include <string>
 #include <thread>
+#include <type_traits>
 #include <vector>
 
 #include <dirent.h>
@@ -245,6 +246,76 @@
   return GetSubdirs(kDeNDataDir);
 }
 
+// Returns first path between |first_dir| and |second_dir| that correspond to a
+// existing directory. Returns error if neither |first_dir| nor |second_dir|
+// correspond to an existing directory.
+inline Result<std::string> FindFirstExistingDirectory(
+    const std::string& first_dir, const std::string& second_dir) {
+  struct stat stat_buf;
+  if (stat(first_dir.c_str(), &stat_buf) != 0) {
+    PLOG(WARNING) << "Failed to stat " << first_dir;
+    if (stat(second_dir.c_str(), &stat_buf) != 0) {
+      return ErrnoError() << "Failed to stat " << second_dir;
+    }
+    if (!S_ISDIR(stat_buf.st_mode)) {
+      return Error() << second_dir << " is not a directory";
+    }
+    return second_dir;
+  }
+
+  if (S_ISDIR(stat_buf.st_mode)) {
+    return first_dir;
+  }
+  LOG(WARNING) << first_dir << " is not a directory";
+
+  if (stat(second_dir.c_str(), &stat_buf) != 0) {
+    return ErrnoError() << "Failed to stat " << second_dir;
+  }
+  if (!S_ISDIR(stat_buf.st_mode)) {
+    return Error() << second_dir << " is not a directory";
+  }
+  return second_dir;
+}
+
+// Copies all entries under |from| directory to |to| directory, and then them.
+// Leaving |from| empty.
+inline Result<void> MoveDir(const std::string& from, const std::string& to) {
+  struct stat stat_buf;
+  if (stat(to.c_str(), &stat_buf) != 0) {
+    return ErrnoError() << "Failed to stat " << to;
+  }
+  if (!S_ISDIR(stat_buf.st_mode)) {
+    return Error() << to << " is not a directory";
+  }
+
+  namespace fs = std::filesystem;
+  std::error_code ec;
+  auto it = fs::directory_iterator(from, ec);
+  if (ec) {
+    return Error() << "Can't read " << from << " : " << ec.message();
+  }
+
+  for (const auto& end = fs::directory_iterator(); it != end;) {
+    auto from_path = it->path();
+    it.increment(ec);
+    if (ec) {
+      return Error() << "Can't read " << from << " : " << ec.message();
+    }
+    auto to_path = to / from_path.filename();
+    fs::copy(from_path, to_path, fs::copy_options::recursive, ec);
+    if (ec) {
+      return Error() << "Failed to copy " << from_path << " to " << to_path
+                     << " : " << ec.message();
+    }
+    fs::remove_all(from_path, ec);
+    if (ec) {
+      return Error() << "Failed to delete " << from_path << " : "
+                     << ec.message();
+    }
+  }
+  return {};
+}
+
 }  // namespace apex
 }  // namespace android
 
diff --git a/apexd/apexd_utils_test.cpp b/apexd/apexd_utils_test.cpp
new file mode 100644
index 0000000..06b9c46
--- /dev/null
+++ b/apexd/apexd_utils_test.cpp
@@ -0,0 +1,183 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <filesystem>
+#include <string>
+
+#include <errno.h>
+
+#include <android-base/file.h>
+#include <android-base/result.h>
+#include <android-base/stringprintf.h>
+#include <gtest/gtest.h>
+
+#include "apexd_test_utils.h"
+#include "apexd_utils.h"
+
+namespace android {
+namespace apex {
+namespace {
+
+using android::apex::testing::IsOk;
+using android::base::Basename;
+using android::base::StringPrintf;
+using ::testing::UnorderedElementsAre;
+using ::testing::UnorderedElementsAreArray;
+
+// TODO(b/170327382): add unit tests for apexd_utils.h
+
+TEST(ApexdUtilTest, FindFirstExistingDirectoryBothExist) {
+  TemporaryDir first_dir;
+  TemporaryDir second_dir;
+  auto result = FindFirstExistingDirectory(first_dir.path, second_dir.path);
+  ASSERT_TRUE(IsOk(result));
+  ASSERT_EQ(*result, first_dir.path);
+}
+
+TEST(ApexdUtilTest, FindFirstExistingDirectoryOnlyFirstExist) {
+  TemporaryDir first_dir;
+  auto second_dir = "/data/local/tmp/does/not/exist";
+  auto result = FindFirstExistingDirectory(first_dir.path, second_dir);
+  ASSERT_TRUE(IsOk(result));
+  ASSERT_EQ(*result, first_dir.path);
+}
+
+TEST(ApexdUtilTest, FindFirstExistingDirectoryOnlySecondExist) {
+  auto first_dir = "/data/local/tmp/does/not/exist";
+  TemporaryDir second_dir;
+  auto result = FindFirstExistingDirectory(first_dir, second_dir.path);
+  ASSERT_TRUE(IsOk(result));
+  ASSERT_EQ(*result, second_dir.path);
+}
+
+TEST(ApexdUtilTest, FindFirstExistingDirectoryNoneExist) {
+  auto first_dir = "/data/local/tmp/does/not/exist";
+  auto second_dir = "/data/local/tmp/also/does/not/exist";
+  auto result = FindFirstExistingDirectory(first_dir, second_dir);
+  ASSERT_FALSE(IsOk(result));
+}
+
+TEST(ApexdUtilTest, FindFirstExistingDirectoryFirstFileSecondDir) {
+  TemporaryFile first_file;
+  TemporaryDir second_dir;
+  auto result = FindFirstExistingDirectory(first_file.path, second_dir.path);
+  ASSERT_TRUE(IsOk(result));
+  ASSERT_EQ(*result, second_dir.path);
+}
+
+TEST(ApexdUtilTest, FindFirstExistingDirectoryFirstDirSecondFile) {
+  TemporaryDir first_dir;
+  TemporaryFile second_file;
+  auto result = FindFirstExistingDirectory(first_dir.path, second_file.path);
+  ASSERT_TRUE(IsOk(result));
+  ASSERT_EQ(*result, first_dir.path);
+}
+
+TEST(ApexdUtilTest, FindFirstExistingDirectoryBothFiles) {
+  TemporaryFile first_file;
+  TemporaryFile second_file;
+  auto result = FindFirstExistingDirectory(first_file.path, second_file.path);
+  ASSERT_FALSE(IsOk(result));
+}
+
+TEST(ApexdUtilTest, FindFirstExistingDirectoryFirstFileSecondDoesNotExist) {
+  TemporaryFile first_file;
+  auto second_dir = "/data/local/tmp/does/not/exist";
+  auto result = FindFirstExistingDirectory(first_file.path, second_dir);
+  ASSERT_FALSE(IsOk(result));
+}
+
+TEST(ApexdUtilTest, FindFirstExistingDirectoryFirsDoesNotExistSecondFile) {
+  auto first_dir = "/data/local/tmp/does/not/exist";
+  TemporaryFile second_file;
+  auto result = FindFirstExistingDirectory(first_dir, second_file.path);
+  ASSERT_FALSE(IsOk(result));
+}
+
+TEST(ApexdUtilTest, MoveDir) {
+  namespace fs = std::filesystem;
+
+  TemporaryDir from;
+  TemporaryDir to;
+
+  TemporaryFile from_1(from.path);
+  auto from_subdir = StringPrintf("%s/subdir", from.path);
+  if (mkdir(from_subdir.c_str(), 07000) != 0) {
+    FAIL() << "Failed to mkdir " << from_subdir << " : " << strerror(errno);
+  }
+  TemporaryFile from_2(from_subdir);
+
+  auto result = MoveDir(from.path, to.path);
+  ASSERT_TRUE(IsOk(result));
+  ASSERT_TRUE(fs::is_empty(from.path));
+
+  std::vector<std::string> content;
+  for (const auto& it : fs::recursive_directory_iterator(to.path)) {
+    content.push_back(it.path());
+  }
+
+  static const std::vector<std::string> expected = {
+      StringPrintf("%s/%s", to.path, Basename(from_1.path).c_str()),
+      StringPrintf("%s/subdir", to.path),
+      StringPrintf("%s/subdir/%s", to.path, Basename(from_2.path).c_str()),
+  };
+  ASSERT_THAT(content, UnorderedElementsAreArray(expected));
+}
+
+TEST(ApexdUtilTest, MoveDirFromIsNotDirectory) {
+  TemporaryFile from;
+  TemporaryDir to;
+  ASSERT_FALSE(IsOk(MoveDir(from.path, to.path)));
+}
+
+TEST(ApexdUtilTest, MoveDirToIsNotDirectory) {
+  TemporaryDir from;
+  TemporaryFile to;
+  TemporaryFile from_1(from.path);
+  ASSERT_FALSE(IsOk(MoveDir(from.path, to.path)));
+}
+
+TEST(ApexdUtilTest, MoveDirFromDoesNotExist) {
+  TemporaryDir to;
+  ASSERT_FALSE(IsOk(MoveDir("/data/local/tmp/does/not/exist", to.path)));
+}
+
+TEST(ApexdUtilTest, MoveDirToDoesNotExist) {
+  namespace fs = std::filesystem;
+
+  TemporaryDir from;
+  TemporaryFile from_1(from.path);
+  auto from_subdir = StringPrintf("%s/subdir", from.path);
+  if (mkdir(from_subdir.c_str(), 07000) != 0) {
+    FAIL() << "Failed to mkdir " << from_subdir << " : " << strerror(errno);
+  }
+  TemporaryFile from_2(from_subdir);
+
+  ASSERT_FALSE(IsOk(MoveDir(from.path, "/data/local/tmp/does/not/exist")));
+
+  // Check that |from| directory is not empty.
+  std::vector<std::string> content;
+  for (const auto& it : fs::recursive_directory_iterator(from.path)) {
+    content.push_back(it.path());
+  }
+
+  ASSERT_THAT(content,
+              UnorderedElementsAre(from_1.path, from_subdir, from_2.path));
+}
+
+}  // namespace
+}  // namespace apex
+}  // namespace android
diff --git a/apexd/apexservice_test.cpp b/apexd/apexservice_test.cpp
index 7f78610..ab93414 100644
--- a/apexd/apexservice_test.cpp
+++ b/apexd/apexservice_test.cpp
@@ -94,6 +94,26 @@
 
 namespace fs = std::filesystem;
 
+static void CleanDir(const std::string& dir) {
+  if (access(dir.c_str(), F_OK) != 0 && errno == ENOENT) {
+    LOG(WARNING) << dir << " does not exist";
+    return;
+  }
+  auto status = WalkDir(dir, [](const fs::directory_entry& p) {
+    std::error_code ec;
+    fs::file_status status = p.status(ec);
+    ASSERT_FALSE(ec) << "Failed to stat " << p.path() << " : " << ec.message();
+    if (fs::is_directory(status)) {
+      fs::remove_all(p.path(), ec);
+    } else {
+      fs::remove(p.path(), ec);
+    }
+    ASSERT_FALSE(ec) << "Failed to delete " << p.path() << " : "
+                     << ec.message();
+  });
+  ASSERT_TRUE(IsOk(status));
+}
+
 class ApexServiceTest : public ::testing::Test {
  public:
   ApexServiceTest() {
@@ -447,21 +467,10 @@
 
  private:
   void CleanUp() {
-    auto status = WalkDir(kApexDataDir, [](const fs::directory_entry& p) {
-      std::error_code ec;
-      fs::file_status status = p.status(ec);
-      ASSERT_FALSE(ec) << "Failed to stat " << p.path() << " : "
-                       << ec.message();
-      if (fs::is_directory(status)) {
-        fs::remove_all(p.path(), ec);
-      } else {
-        fs::remove(p.path(), ec);
-      }
-      ASSERT_FALSE(ec) << "Failed to delete " << p.path() << " : "
-                       << ec.message();
-    });
-    fs::remove_all(kApexSessionsDir);
-    ASSERT_TRUE(IsOk(status));
+    CleanDir(kActiveApexPackagesDataDir);
+    CleanDir(kApexBackupDir);
+    CleanDir(kApexHashTreeDir);
+    CleanDir(ApexSession::GetSessionsDir());
 
     DeleteIfExists("/data/misc_ce/0/apexdata/apex.apexd_test");
     DeleteIfExists("/data/misc_ce/0/apexrollback/123456");
diff --git a/tests/src/com/android/tests/apex/ApexdHostTest.java b/tests/src/com/android/tests/apex/ApexdHostTest.java
index fc1fa4b..ac18357 100644
--- a/tests/src/com/android/tests/apex/ApexdHostTest.java
+++ b/tests/src/com/android/tests/apex/ApexdHostTest.java
@@ -196,4 +196,32 @@
             getDevice().executeShellV2Command("rm /data/apex/active/apex.apexd_test_v2_no_pb.apex");
         }
     }
+
+    /**
+     * Verifies that content of {@code /data/apex/sessions/} is migrated to the {@code
+     * /metadata/apex/sessions}.
+     */
+    @Test
+    public void testSessionsDirMigrationToMetadata() throws Exception {
+        assumeTrue("Device does not support updating APEX", mTestUtils.isApexUpdateSupported());
+        assumeTrue("Device requires root", getDevice().isAdbRoot());
+
+        try {
+            getDevice().executeShellV2Command("mkdir -p /data/apex/sessions/1543");
+            File file = File.createTempFile("foo", "bar");
+            getDevice().pushFile(file, "/data/apex/sessions/1543/file");
+
+            // During boot sequence apexd will move /data/apex/sessions/1543/file to
+            // /metadata/apex/sessions/1543/file.
+            getDevice().reboot();
+            assertWithMessage("Timed out waiting for device to boot").that(
+                    getDevice().waitForBootComplete(Duration.ofMinutes(2).toMillis())).isTrue();
+
+            assertThat(getDevice().doesFileExist("/metadata/apex/sessions/1543/file")).isTrue();
+            assertThat(getDevice().doesFileExist("/data/apex/sessions/1543/file")).isFalse();
+        } finally {
+            getDevice().executeShellV2Command("rm -R /data/apex/sessions/1543");
+            getDevice().executeShellV2Command("rm -R /metadata/apex/sessions/1543");
+        }
+    }
 }