Prepare app data only when storage is available.

Before this change, scanning a package aggressively tried checking
to ensure that private app data was prepared.  However, in an FBE
world we may not have access to that data at scan time.  So this
change shifts the preparing of private app data until later: it
prepares DE storage when a user is started, and CE storage when a
user is unlocked.  Wire ourselves into the user lifecycle so we can
prepare storage at both user start and unlock.

When DE/CE storage becomes available, this change reconciles any
found packages against known installed apps, and deletes any orphaned
data directories.

We now need to store the last-restorecon hash in an xattr on a
per-user directory basis, since we can't restorecon CE storage until
it's unlocked, or adopted storage until it's mounted.  Remove a
bunch of used logic for loading dynamic SELinux policy at runtime;
our policy always comes from the system image.

Bug: 26466827, 26544104
Change-Id: I8d0a4ef862c35f4e4ef5c7f20d3bb8f12ba3fd4b
diff --git a/core/java/android/os/UserManager.java b/core/java/android/os/UserManager.java
index d1b3f59..a95602c 100644
--- a/core/java/android/os/UserManager.java
+++ b/core/java/android/os/UserManager.java
@@ -766,9 +766,13 @@
      * @param user The user to retrieve the running state for.
      */
     public boolean isUserRunning(UserHandle user) {
+        return isUserRunning(user.getIdentifier());
+    }
+
+    /** {@hide} */
+    public boolean isUserRunning(int userId) {
         try {
-            return ActivityManagerNative.getDefault().isUserRunning(
-                    user.getIdentifier(), 0);
+            return ActivityManagerNative.getDefault().isUserRunning(userId, 0);
         } catch (RemoteException e) {
             return false;
         }
diff --git a/services/core/java/com/android/server/am/ActivityManagerService.java b/services/core/java/com/android/server/am/ActivityManagerService.java
index ca38b71..5355e70 100644
--- a/services/core/java/com/android/server/am/ActivityManagerService.java
+++ b/services/core/java/com/android/server/am/ActivityManagerService.java
@@ -12684,7 +12684,6 @@
 
         if (goingCallback != null) goingCallback.run();
 
-
         mBatteryStatsService.noteEvent(BatteryStats.HistoryItem.EVENT_USER_RUNNING_START,
                 Integer.toString(currentUserId), currentUserId);
         mBatteryStatsService.noteEvent(BatteryStats.HistoryItem.EVENT_USER_FOREGROUND_START,
diff --git a/services/core/java/com/android/server/am/UserController.java b/services/core/java/com/android/server/am/UserController.java
index 56757ca..a0e084b 100644
--- a/services/core/java/com/android/server/am/UserController.java
+++ b/services/core/java/com/android/server/am/UserController.java
@@ -249,6 +249,9 @@
             if (uss.state == UserState.STATE_RUNNING_LOCKED) {
                 uss.setState(UserState.STATE_RUNNING);
 
+                // Give user manager a chance to prepare app storage
+                mUserManager.onBeforeUnlockUser(userId);
+
                 mHandler.sendMessage(mHandler.obtainMessage(SYSTEM_USER_UNLOCK_MSG, userId, 0));
 
                 final Intent unlockedIntent = new Intent(Intent.ACTION_USER_UNLOCKED);
@@ -651,7 +654,8 @@
                 }
 
                 if (uss.state == UserState.STATE_BOOTING) {
-                    // Let user manager propagate user restrictions to other services.
+                    // Give user manager a chance to propagate user restrictions
+                    // to other services and prepare app storage
                     getUserManager().onBeforeStartUser(userId);
 
                     // Booting up a new user, need to tell system services about it.
diff --git a/services/core/java/com/android/server/pm/Installer.java b/services/core/java/com/android/server/pm/Installer.java
index 190eca6..508bf91 100644
--- a/services/core/java/com/android/server/pm/Installer.java
+++ b/services/core/java/com/android/server/pm/Installer.java
@@ -16,6 +16,7 @@
 
 package com.android.server.pm;
 
+import android.annotation.IntDef;
 import android.annotation.Nullable;
 import android.content.Context;
 import android.content.pm.PackageStats;
@@ -28,6 +29,9 @@
 
 import dalvik.system.VMRuntime;
 
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
 public final class Installer extends SystemService {
     private static final String TAG = "Installer";
 
@@ -46,8 +50,17 @@
     /** Run the application with the JIT compiler */
     public static final int DEXOPT_USEJIT       = 1 << 5;
 
+    /** @hide */
+    @IntDef(flag = true, value = {
+            FLAG_DE_STORAGE,
+            FLAG_CE_STORAGE,
+    })
+    @Retention(RetentionPolicy.SOURCE)
+    public @interface StorageFlags {}
+
     public static final int FLAG_DE_STORAGE = 1 << 0;
     public static final int FLAG_CE_STORAGE = 1 << 1;
+
     public static final int FLAG_CLEAR_CACHE_ONLY = 1 << 2;
     public static final int FLAG_CLEAR_CODE_CACHE_ONLY = 1 << 3;
 
diff --git a/services/core/java/com/android/server/pm/PackageManagerException.java b/services/core/java/com/android/server/pm/PackageManagerException.java
index d04eedc..f243e63 100644
--- a/services/core/java/com/android/server/pm/PackageManagerException.java
+++ b/services/core/java/com/android/server/pm/PackageManagerException.java
@@ -25,6 +25,11 @@
 public class PackageManagerException extends Exception {
     public final int error;
 
+    public PackageManagerException(String detailMessage) {
+        super(detailMessage);
+        this.error = PackageManager.INSTALL_FAILED_INTERNAL_ERROR;
+    }
+
     public PackageManagerException(int error, String detailMessage) {
         super(detailMessage);
         this.error = error;
diff --git a/services/core/java/com/android/server/pm/PackageManagerService.java b/services/core/java/com/android/server/pm/PackageManagerService.java
index c1cba97..f6fa8bf 100644
--- a/services/core/java/com/android/server/pm/PackageManagerService.java
+++ b/services/core/java/com/android/server/pm/PackageManagerService.java
@@ -46,7 +46,6 @@
 import static android.content.pm.PackageManager.INSTALL_FAILED_REPLACE_COULDNT_DELETE;
 import static android.content.pm.PackageManager.INSTALL_FAILED_SHARED_USER_INCOMPATIBLE;
 import static android.content.pm.PackageManager.INSTALL_FAILED_TEST_ONLY;
-import static android.content.pm.PackageManager.INSTALL_FAILED_UID_CHANGED;
 import static android.content.pm.PackageManager.INSTALL_FAILED_UPDATE_INCOMPATIBLE;
 import static android.content.pm.PackageManager.INSTALL_FAILED_USER_RESTRICTED;
 import static android.content.pm.PackageManager.INSTALL_FAILED_VERSION_DOWNGRADE;
@@ -192,7 +191,6 @@
 import android.security.SystemKeyStore;
 import android.system.ErrnoException;
 import android.system.Os;
-import android.system.StructStat;
 import android.text.TextUtils;
 import android.text.format.DateUtils;
 import android.util.ArrayMap;
@@ -234,6 +232,7 @@
 import com.android.server.ServiceThread;
 import com.android.server.SystemConfig;
 import com.android.server.Watchdog;
+import com.android.server.pm.Installer.StorageFlags;
 import com.android.server.pm.PermissionsState.PermissionState;
 import com.android.server.pm.Settings.DatabaseVersion;
 import com.android.server.pm.Settings.VersionInfo;
@@ -316,6 +315,7 @@
     private static final boolean DEBUG_ABI_SELECTION = false;
     private static final boolean DEBUG_EPHEMERAL = false;
     private static final boolean DEBUG_TRIAGED_MISSING = false;
+    private static final boolean DEBUG_APP_DATA = false;
 
     static final boolean CLEAR_RUNTIME_PERMISSIONS_ON_UPGRADE = false;
 
@@ -518,9 +518,6 @@
     // If mac_permissions.xml was found for seinfo labeling.
     boolean mFoundPolicyFile;
 
-    // If a recursive restorecon of /data/data/<pkg> is needed.
-    private boolean mShouldRestoreconData = SELinuxMMAC.shouldRestorecon();
-
     private final EphemeralApplicationRegistry mEphemeralApplicationRegistry;
 
     public static final class SharedLibraryEntry {
@@ -1750,7 +1747,7 @@
         @Override
         public void onVolumeForgotten(String fsUuid) {
             if (TextUtils.isEmpty(fsUuid)) {
-                Slog.w(TAG, "Forgetting internal storage is probably a mistake; ignoring");
+                Slog.e(TAG, "Forgetting internal storage is probably a mistake; ignoring");
                 return;
             }
 
@@ -2342,6 +2339,17 @@
                 }
             }
 
+            // Prepare storage for system user really early during boot,
+            // since core system apps like SettingsProvider and SystemUI
+            // can't wait for user to start
+            final int flags;
+            if (StorageManager.isFileBasedEncryptionEnabled()) {
+                flags = Installer.FLAG_DE_STORAGE;
+            } else {
+                flags = Installer.FLAG_DE_STORAGE | Installer.FLAG_CE_STORAGE;
+            }
+            reconcileAppsData(StorageManager.UUID_PRIVATE_INTERNAL, UserHandle.USER_SYSTEM, flags);
+
             // If this is first boot after an OTA, and a normal boot, then
             // we need to clear code cache directories.
             if (mIsUpgrade && !onlyCore) {
@@ -6719,23 +6727,6 @@
         return true;
     }
 
-    private void createDataDirsLI(String volumeUuid, String packageName, int uid, String seinfo)
-            throws PackageManagerException {
-        // TODO: triage flags as part of 26466827
-        final int appId = UserHandle.getAppId(uid);
-        final int flags = Installer.FLAG_CE_STORAGE | Installer.FLAG_DE_STORAGE;
-
-        try {
-            final int[] users = sUserManager.getUserIds();
-            for (int user : users) {
-                mInstaller.createAppData(volumeUuid, packageName, user, flags, appId, seinfo);
-            }
-        } catch (InstallerException e) {
-            throw new PackageManagerException(INSTALL_FAILED_INSUFFICIENT_STORAGE,
-                    "Failed to prepare data directory", e);
-        }
-    }
-
     private boolean removeDataDirsLI(String volumeUuid, String packageName) {
         // TODO: triage flags as part of 26466827
         final int flags = Installer.FLAG_CE_STORAGE | Installer.FLAG_DE_STORAGE;
@@ -6765,6 +6756,23 @@
         }
     }
 
+    void destroyAppDataLI(String volumeUuid, String packageName, int userId, int flags) {
+        try {
+            mInstaller.destroyAppData(volumeUuid, packageName, userId, flags);
+        } catch (InstallerException e) {
+            Slog.w(TAG, "Failed to destroy app data", e);
+        }
+    }
+
+    void restoreconAppDataLI(String volumeUuid, String packageName, int userId, int flags,
+            int appId, String seinfo) {
+        try {
+            mInstaller.restoreconAppData(volumeUuid, packageName, userId, flags, appId, seinfo);
+        } catch (InstallerException e) {
+            Slog.e(TAG, "Failed to restorecon for " + packageName + ": " + e);
+        }
+    }
+
     private void deleteCodeCacheDirsLI(String volumeUuid, String packageName) {
         // TODO: triage flags as part of 26466827
         final int flags = Installer.FLAG_CE_STORAGE | Installer.FLAG_DE_STORAGE;
@@ -7274,104 +7282,8 @@
                 pkg.applicationInfo.uid);
 
         if (pkg != mPlatformPackage) {
-            // This is a normal package, need to make its data directory.
-            final File dataPath = Environment.getDataUserCredentialEncryptedPackageDirectory(
-                    pkg.volumeUuid, UserHandle.USER_SYSTEM, pkg.packageName);
-
-            // TOOD: switch to ensure various directories
-
-            boolean uidError = false;
-            if (dataPath.exists()) {
-                int currentUid = 0;
-                try {
-                    StructStat stat = Os.stat(dataPath.getPath());
-                    currentUid = stat.st_uid;
-                } catch (ErrnoException e) {
-                    Slog.e(TAG, "Couldn't stat path " + dataPath.getPath(), e);
-                }
-
-                // If we have mismatched owners for the data path, we have a problem.
-                if (currentUid != pkg.applicationInfo.uid) {
-                    boolean recovered = false;
-                    if (((parseFlags & PackageParser.PARSE_IS_SYSTEM) != 0
-                            || (scanFlags & SCAN_BOOTING) != 0)) {
-                        // If this is a system app, we can at least delete its
-                        // current data so the application will still work.
-                        boolean res = removeDataDirsLI(pkg.volumeUuid, pkgName);
-                        if (res) {
-                            // TODO: Kill the processes first
-                            // Old data gone!
-                            String prefix = (parseFlags&PackageParser.PARSE_IS_SYSTEM) != 0
-                                    ? "System package " : "Third party package ";
-                            String msg = prefix + pkg.packageName
-                                    + " has changed from uid: "
-                                    + currentUid + " to "
-                                    + pkg.applicationInfo.uid + "; old data erased";
-                            reportSettingsProblem(Log.WARN, msg);
-                            recovered = true;
-                        }
-                        if (!recovered) {
-                            mHasSystemUidErrors = true;
-                        }
-                    } else {
-                        // If we allow this install to proceed, we will be broken.
-                        // Abort, abort!
-                        throw new PackageManagerException(INSTALL_FAILED_UID_CHANGED,
-                                "Expected data to be owned by UID " + pkg.applicationInfo.uid
-                                        + " but found " + currentUid);
-                    }
-                    if (!recovered) {
-                        pkg.applicationInfo.dataDir = "/mismatched_uid/settings_"
-                            + pkg.applicationInfo.uid + "/fs_"
-                            + currentUid;
-                        pkg.applicationInfo.nativeLibraryDir = pkg.applicationInfo.dataDir;
-                        pkg.applicationInfo.nativeLibraryRootDir = pkg.applicationInfo.dataDir;
-                        String msg = "Package " + pkg.packageName
-                                + " has mismatched uid: "
-                                + currentUid + " on disk, "
-                                + pkg.applicationInfo.uid + " in settings";
-                        // writer
-                        synchronized (mPackages) {
-                            mSettings.mReadMessages.append(msg);
-                            mSettings.mReadMessages.append('\n');
-                            uidError = true;
-                            if (!pkgSetting.uidError) {
-                                reportSettingsProblem(Log.ERROR, msg);
-                            }
-                        }
-                    }
-                }
-
-                // Ensure that directories are prepared
-                createDataDirsLI(pkg.volumeUuid, pkgName, pkg.applicationInfo.uid,
-                        pkg.applicationInfo.seinfo);
-
-                if (mShouldRestoreconData) {
-                    Slog.i(TAG, "SELinux relabeling of " + pkg.packageName + " issued.");
-                    // TODO: extend this to restorecon over all users
-                    final int appId = UserHandle.getAppId(pkg.applicationInfo.uid);
-                    // TODO: triage flags as part of 26466827
-                    final int flags = Installer.FLAG_CE_STORAGE | Installer.FLAG_DE_STORAGE;
-                    try {
-                        mInstaller.restoreconAppData(pkg.volumeUuid, pkg.packageName,
-                                UserHandle.USER_SYSTEM, flags, appId, pkg.applicationInfo.seinfo);
-                    } catch (InstallerException e) {
-                        Slog.w(TAG, "Failed to restorecon " + pkg.packageName, e);
-                    }
-                }
-            } else {
-                if (DEBUG_PACKAGE_SCANNING) {
-                    if ((parseFlags & PackageParser.PARSE_CHATTY) != 0)
-                        Log.v(TAG, "Want this data dir: " + dataPath);
-                }
-                createDataDirsLI(pkg.volumeUuid, pkgName, pkg.applicationInfo.uid,
-                        pkg.applicationInfo.seinfo);
-            }
-
             // Get all of our default paths setup
             pkg.applicationInfo.initForUser(UserHandle.USER_SYSTEM);
-
-            pkgSetting.uidError = uidError;
         }
 
         final String path = scanFile.getPath();
@@ -7405,49 +7317,6 @@
             setNativeLibraryPaths(pkg);
         }
 
-        if (DEBUG_INSTALL) Slog.i(TAG, "Linking native library dir for " + path);
-        final int[] userIds = sUserManager.getUserIds();
-        synchronized (mInstallLock) {
-            // Make sure all user data directories are ready to roll; we're okay
-            // if they already exist
-            if (!TextUtils.isEmpty(pkg.volumeUuid)) {
-                for (int userId : userIds) {
-                    if (userId != UserHandle.USER_SYSTEM) {
-                        // TODO: triage flags as part of 26466827
-                        final int flags = Installer.FLAG_CE_STORAGE | Installer.FLAG_DE_STORAGE;
-                        final int appId = UserHandle.getAppId(pkg.applicationInfo.uid);
-                        try {
-                            mInstaller.createAppData(pkg.volumeUuid, pkg.packageName, userId,
-                                    flags, appId, pkg.applicationInfo.seinfo);
-                        } catch (InstallerException e) {
-                            throw PackageManagerException.from(e);
-                        }
-                    }
-                }
-            }
-
-            // Create a native library symlink only if we have native libraries
-            // and if the native libraries are 32 bit libraries. We do not provide
-            // this symlink for 64 bit libraries.
-            if (pkg.applicationInfo.primaryCpuAbi != null &&
-                    !VMRuntime.is64BitAbi(pkg.applicationInfo.primaryCpuAbi)) {
-                Trace.traceBegin(TRACE_TAG_PACKAGE_MANAGER, "linkNativeLib");
-                try {
-                    final String nativeLibPath = pkg.applicationInfo.nativeLibraryDir;
-                    for (int userId : userIds) {
-                        try {
-                            mInstaller.linkNativeLibraryDirectory(pkg.volumeUuid, pkg.packageName,
-                                    nativeLibPath, userId);
-                        } catch (InstallerException e) {
-                            throw PackageManagerException.from(e);
-                        }
-                    }
-                } finally {
-                    Trace.traceEnd(TRACE_TAG_PACKAGE_MANAGER);
-                }
-            }
-        }
-
         // This is a special case for the "system" package, where the ABI is
         // dictated by the zygote configuration (and init.rc). We should keep track
         // of this ABI so that we can deal with "normal" applications that run under
@@ -10228,21 +10097,14 @@
                     pkgSetting.setInstalled(true, userId);
                     pkgSetting.setHidden(false, userId);
                     mSettings.writePackageRestrictionsLPr(userId);
+                    if (pkgSetting.pkg != null) {
+                        prepareAppDataAfterInstall(pkgSetting.pkg);
+                    }
                     installed = true;
                 }
             }
 
             if (installed) {
-                synchronized (mInstallLock) {
-                    final int flags = Installer.FLAG_DE_STORAGE | Installer.FLAG_CE_STORAGE;
-                    try {
-                        mInstaller.createAppData(pkgSetting.volumeUuid, packageName, userId, flags,
-                                pkgSetting.appId, pkgSetting.pkg.applicationInfo.seinfo);
-                    } catch (InstallerException e) {
-                        throw new IllegalStateException(e);
-                    }
-                }
-
                 sendPackageAddedForUser(packageName, pkgSetting, userId);
             }
         } finally {
@@ -12368,6 +12230,8 @@
                     System.currentTimeMillis(), user);
 
             updateSettingsLI(newPackage, installerPackageName, volumeUuid, null, null, res, user);
+            prepareAppDataAfterInstall(newPackage);
+
             // delete the partially installed application. the data directory will have to be
             // restored if it was already existing
             if (res.returnCode != PackageManager.INSTALL_SUCCEEDED) {
@@ -12524,6 +12388,7 @@
                         scanFlags | SCAN_UPDATE_TIME, System.currentTimeMillis(), user);
                 updateSettingsLI(newPackage, installerPackageName, volumeUuid, allUsers,
                         perUserInstalled, res, user);
+                prepareAppDataAfterInstall(newPackage);
                 updatedSettings = true;
             } catch (PackageManagerException e) {
                 res.setError("Package couldn't be installed in " + pkg.codePath, e);
@@ -12655,6 +12520,7 @@
             if (res.returnCode == PackageManager.INSTALL_SUCCEEDED) {
                 updateSettingsLI(newPackage, installerPackageName, volumeUuid, allUsers,
                         perUserInstalled, res, user);
+                prepareAppDataAfterInstall(newPackage);
                 updatedSettings = true;
             }
 
@@ -13692,6 +13558,8 @@
             return false;
         }
 
+        prepareAppDataAfterInstall(newPkg);
+
         // writer
         synchronized (mPackages) {
             PackageSetting ps = mSettings.mPackages.get(newPkg.packageName);
@@ -16179,10 +16047,6 @@
      */
     public void scanAvailableAsecs() {
         updateExternalMediaStatusInner(true, false, false);
-        if (mShouldRestoreconData) {
-            SELinuxMMAC.setRestoreconDone();
-            mShouldRestoreconData = false;
-        }
     }
 
     /*
@@ -16503,22 +16367,31 @@
     }
 
     private void loadPrivatePackagesInner(VolumeInfo vol) {
+        final String volumeUuid = vol.fsUuid;
+        if (TextUtils.isEmpty(volumeUuid)) {
+            Slog.e(TAG, "Loading internal storage is probably a mistake; ignoring");
+            return;
+        }
+
         final ArrayList<ApplicationInfo> loaded = new ArrayList<>();
         final int parseFlags = mDefParseFlags | PackageParser.PARSE_EXTERNAL_STORAGE;
 
         final VersionInfo ver;
         final List<PackageSetting> packages;
         synchronized (mPackages) {
-            ver = mSettings.findOrCreateVersion(vol.fsUuid);
-            packages = mSettings.getVolumePackagesLPr(vol.fsUuid);
+            ver = mSettings.findOrCreateVersion(volumeUuid);
+            packages = mSettings.getVolumePackagesLPr(volumeUuid);
         }
 
+        // TODO: introduce a new concept similar to "frozen" to prevent these
+        // apps from being launched until after data has been fully reconciled
         for (PackageSetting ps : packages) {
             synchronized (mInstallLock) {
                 final PackageParser.Package pkg;
                 try {
                     pkg = scanPackageTracedLI(ps.codePath, parseFlags, SCAN_INITIAL, 0, null);
                     loaded.add(pkg.applicationInfo);
+
                 } catch (PackageManagerException e) {
                     Slog.w(TAG, "Failed to scan " + ps.codePath + ": " + e.getMessage());
                 }
@@ -16529,14 +16402,27 @@
             }
         }
 
+        // Reconcile app data for all started/unlocked users
+        final UserManager um = mContext.getSystemService(UserManager.class);
+        for (UserInfo user : um.getUsers()) {
+            if (um.isUserUnlocked(user.id)) {
+                reconcileAppsData(volumeUuid, user.id,
+                        Installer.FLAG_DE_STORAGE | Installer.FLAG_CE_STORAGE);
+            } else if (um.isUserRunning(user.id)) {
+                reconcileAppsData(volumeUuid, user.id, Installer.FLAG_DE_STORAGE);
+            } else {
+                continue;
+            }
+        }
+
         synchronized (mPackages) {
             int updateFlags = UPDATE_PERMISSIONS_ALL;
             if (ver.sdkVersion != mSdkVersion) {
                 logCriticalInfo(Log.INFO, "Platform changed from " + ver.sdkVersion + " to "
-                        + mSdkVersion + "; regranting permissions for " + vol.fsUuid);
+                        + mSdkVersion + "; regranting permissions for " + volumeUuid);
                 updateFlags |= UPDATE_PERMISSIONS_REPLACE_PKG | UPDATE_PERMISSIONS_REPLACE_ALL;
             }
-            updatePermissionsLPw(null, null, vol.fsUuid, updateFlags);
+            updatePermissionsLPw(null, null, volumeUuid, updateFlags);
 
             // Yay, everything is now upgraded
             ver.forceCurrent();
@@ -16558,10 +16444,16 @@
     }
 
     private void unloadPrivatePackagesInner(VolumeInfo vol) {
+        final String volumeUuid = vol.fsUuid;
+        if (TextUtils.isEmpty(volumeUuid)) {
+            Slog.e(TAG, "Unloading internal storage is probably a mistake; ignoring");
+            return;
+        }
+
         final ArrayList<ApplicationInfo> unloaded = new ArrayList<>();
         synchronized (mInstallLock) {
         synchronized (mPackages) {
-            final List<PackageSetting> packages = mSettings.getVolumePackagesLPr(vol.fsUuid);
+            final List<PackageSetting> packages = mSettings.getVolumePackagesLPr(volumeUuid);
             for (PackageSetting ps : packages) {
                 if (ps.pkg == null) continue;
 
@@ -16645,6 +16537,37 @@
         }
     }
 
+    private void assertPackageKnown(String volumeUuid, String packageName)
+            throws PackageManagerException {
+        synchronized (mPackages) {
+            final PackageSetting ps = mSettings.mPackages.get(packageName);
+            if (ps == null) {
+                throw new PackageManagerException("Package " + packageName + " is unknown");
+            } else if (!TextUtils.equals(volumeUuid, ps.volumeUuid)) {
+                throw new PackageManagerException(
+                        "Package " + packageName + " found on unknown volume " + volumeUuid
+                                + "; expected volume " + ps.volumeUuid);
+            }
+        }
+    }
+
+    private void assertPackageKnownAndInstalled(String volumeUuid, String packageName, int userId)
+            throws PackageManagerException {
+        synchronized (mPackages) {
+            final PackageSetting ps = mSettings.mPackages.get(packageName);
+            if (ps == null) {
+                throw new PackageManagerException("Package " + packageName + " is unknown");
+            } else if (!TextUtils.equals(volumeUuid, ps.volumeUuid)) {
+                throw new PackageManagerException(
+                        "Package " + packageName + " found on unknown volume " + volumeUuid
+                                + "; expected volume " + ps.volumeUuid);
+            } else if (!ps.getInstalled(userId)) {
+                throw new PackageManagerException(
+                        "Package " + packageName + " not installed for user " + userId);
+            }
+        }
+    }
+
     /**
      * Examine all apps present on given mounted volume, and destroy apps that
      * aren't expected, either due to uninstallation or reinstallation on
@@ -16661,37 +16584,223 @@
                 continue;
             }
 
-            boolean destroyApp = false;
-            String packageName = null;
             try {
                 final PackageLite pkg = PackageParser.parsePackageLite(file,
                         PackageParser.PARSE_MUST_BE_APK);
-                packageName = pkg.packageName;
+                assertPackageKnown(volumeUuid, pkg.packageName);
 
-                synchronized (mPackages) {
-                    final PackageSetting ps = mSettings.mPackages.get(packageName);
-                    if (ps == null) {
-                        logCriticalInfo(Log.WARN, "Destroying " + packageName + " on + "
-                                + volumeUuid + " because we found no install record");
-                        destroyApp = true;
-                    } else if (!TextUtils.equals(volumeUuid, ps.volumeUuid)) {
-                        logCriticalInfo(Log.WARN, "Destroying " + packageName + " on "
-                                + volumeUuid + " because we expected it on " + ps.volumeUuid);
-                        destroyApp = true;
-                    }
+            } catch (PackageParserException | PackageManagerException e) {
+                logCriticalInfo(Log.WARN, "Destroying " + file + " due to: " + e);
+                synchronized (mInstallLock) {
+                    removeCodePathLI(file);
                 }
+            }
+        }
+    }
 
-            } catch (PackageParserException e) {
-                logCriticalInfo(Log.WARN, "Destroying " + file + " due to parse failure: " + e);
-                destroyApp = true;
+    /**
+     * Reconcile all app data for the given user.
+     * <p>
+     * Verifies that directories exist and that ownership and labeling is
+     * correct for all installed apps on all mounted volumes.
+     */
+    void reconcileAppsData(int userId, @StorageFlags int flags) {
+        final StorageManager storage = mContext.getSystemService(StorageManager.class);
+        for (VolumeInfo vol : storage.getWritablePrivateVolumes()) {
+            final String volumeUuid = vol.getFsUuid();
+            reconcileAppsData(volumeUuid, userId, flags);
+        }
+    }
+
+    /**
+     * Reconcile all app data on given mounted volume.
+     * <p>
+     * Destroys app data that isn't expected, either due to uninstallation or
+     * reinstallation on another volume.
+     * <p>
+     * Verifies that directories exist and that ownership and labeling is
+     * correct for all installed apps.
+     */
+    private void reconcileAppsData(String volumeUuid, int userId, @StorageFlags int flags) {
+        Slog.v(TAG, "reconcileAppsData for " + volumeUuid + " u" + userId + " 0x"
+                + Integer.toHexString(flags));
+
+        final File ceDir = Environment.getDataUserCredentialEncryptedDirectory(volumeUuid, userId);
+        final File deDir = Environment.getDataUserDeviceEncryptedDirectory(volumeUuid, userId);
+
+        boolean restoreconNeeded = false;
+
+        // First look for stale data that doesn't belong, and check if things
+        // have changed since we did our last restorecon
+        if ((flags & Installer.FLAG_CE_STORAGE) != 0) {
+            if (!isUserKeyUnlocked(userId)) {
+                throw new RuntimeException(
+                        "Yikes, someone asked us to reconcile CE storage while " + userId
+                                + " was still locked; this would have caused massive data loss!");
             }
 
-            if (destroyApp) {
-                synchronized (mInstallLock) {
-                    if (packageName != null) {
-                        removeDataDirsLI(volumeUuid, packageName);
+            restoreconNeeded |= SELinuxMMAC.isRestoreconNeeded(ceDir);
+
+            final File[] files = FileUtils.listFilesOrEmpty(ceDir);
+            for (File file : files) {
+                final String packageName = file.getName();
+                try {
+                    assertPackageKnownAndInstalled(volumeUuid, packageName, userId);
+                } catch (PackageManagerException e) {
+                    logCriticalInfo(Log.WARN, "Destroying " + file + " due to: " + e);
+                    synchronized (mInstallLock) {
+                        destroyAppDataLI(volumeUuid, packageName, userId,
+                                Installer.FLAG_CE_STORAGE);
                     }
-                    removeCodePathLI(file);
+                }
+            }
+        }
+        if ((flags & Installer.FLAG_DE_STORAGE) != 0) {
+            restoreconNeeded |= SELinuxMMAC.isRestoreconNeeded(deDir);
+
+            final File[] files = FileUtils.listFilesOrEmpty(deDir);
+            for (File file : files) {
+                final String packageName = file.getName();
+                try {
+                    assertPackageKnownAndInstalled(volumeUuid, packageName, userId);
+                } catch (PackageManagerException e) {
+                    logCriticalInfo(Log.WARN, "Destroying " + file + " due to: " + e);
+                    synchronized (mInstallLock) {
+                        destroyAppDataLI(volumeUuid, packageName, userId,
+                                Installer.FLAG_DE_STORAGE);
+                    }
+                }
+            }
+        }
+
+        // Ensure that data directories are ready to roll for all packages
+        // installed for this volume and user
+        final List<PackageSetting> packages;
+        synchronized (mPackages) {
+            packages = mSettings.getVolumePackagesLPr(volumeUuid);
+        }
+        int preparedCount = 0;
+        for (PackageSetting ps : packages) {
+            final String packageName = ps.name;
+            if (ps.pkg == null) {
+                Slog.w(TAG, "Odd, missing scanned package " + packageName);
+                // TODO: might be due to legacy ASEC apps; we should circle back
+                // and reconcile again once they're scanned
+                continue;
+            }
+
+            if (ps.getInstalled(userId)) {
+                prepareAppData(volumeUuid, userId, flags, ps.pkg, restoreconNeeded);
+                preparedCount++;
+            }
+        }
+
+        if (restoreconNeeded) {
+            if ((flags & Installer.FLAG_CE_STORAGE) != 0) {
+                SELinuxMMAC.setRestoreconDone(ceDir);
+            }
+            if ((flags & Installer.FLAG_DE_STORAGE) != 0) {
+                SELinuxMMAC.setRestoreconDone(deDir);
+            }
+        }
+
+        Slog.v(TAG, "reconcileAppsData finished " + preparedCount
+                + " packages; restoreconNeeded was " + restoreconNeeded);
+    }
+
+    /**
+     * Prepare app data for the given app just after it was installed or
+     * upgraded. This method carefully only touches users that it's installed
+     * for, and it forces a restorecon to handle any seinfo changes.
+     * <p>
+     * Verifies that directories exist and that ownership and labeling is
+     * correct for all installed apps. If there is an ownership mismatch, it
+     * will try recovering system apps by wiping data; third-party app data is
+     * left intact.
+     */
+    private void prepareAppDataAfterInstall(PackageParser.Package pkg) {
+        final PackageSetting ps;
+        synchronized (mPackages) {
+            ps = mSettings.mPackages.get(pkg.packageName);
+        }
+
+        final UserManager um = mContext.getSystemService(UserManager.class);
+        for (UserInfo user : um.getUsers()) {
+            final int flags;
+            if (um.isUserUnlocked(user.id)) {
+                flags = Installer.FLAG_DE_STORAGE | Installer.FLAG_CE_STORAGE;
+            } else if (um.isUserRunning(user.id)) {
+                flags = Installer.FLAG_DE_STORAGE;
+            } else {
+                continue;
+            }
+
+            if (ps.getInstalled(user.id)) {
+                // Whenever an app changes, force a restorecon of its data
+                // TODO: when user data is locked, mark that we're still dirty
+                prepareAppData(pkg.volumeUuid, user.id, flags, pkg, true);
+            }
+        }
+    }
+
+    /**
+     * Prepare app data for the given app.
+     * <p>
+     * Verifies that directories exist and that ownership and labeling is
+     * correct for all installed apps. If there is an ownership mismatch, this
+     * will try recovering system apps by wiping data; third-party app data is
+     * left intact.
+     */
+    private void prepareAppData(String volumeUuid, int userId, @StorageFlags int flags,
+            PackageParser.Package pkg, boolean restoreconNeeded) {
+        if (DEBUG_APP_DATA) {
+            Slog.v(TAG, "prepareAppData for " + pkg.packageName + " u" + userId + " 0x"
+                    + Integer.toHexString(flags) + (restoreconNeeded ? " restoreconNeeded" : ""));
+        }
+
+        final String packageName = pkg.packageName;
+        final ApplicationInfo app = pkg.applicationInfo;
+        final int appId = UserHandle.getAppId(app.uid);
+
+        Preconditions.checkNotNull(app.seinfo);
+
+        synchronized (mInstallLock) {
+            try {
+                mInstaller.createAppData(volumeUuid, packageName, userId, flags,
+                        appId, app.seinfo);
+            } catch (InstallerException e) {
+                if (app.isSystemApp()) {
+                    logCriticalInfo(Log.ERROR, "Failed to create app data for " + packageName
+                            + ", but trying to recover: " + e);
+                    destroyAppDataLI(volumeUuid, packageName, userId, flags);
+                    try {
+                        mInstaller.createAppData(volumeUuid, packageName, userId, flags,
+                                appId, app.seinfo);
+                        logCriticalInfo(Log.DEBUG, "Recovery succeeded!");
+                    } catch (InstallerException e2) {
+                        logCriticalInfo(Log.DEBUG, "Recovery failed!");
+                    }
+                } else {
+                    Slog.e(TAG, "Failed to create app data for " + packageName + ": " + e);
+                }
+            }
+
+            if (restoreconNeeded) {
+                restoreconAppDataLI(volumeUuid, packageName, userId, flags, appId, app.seinfo);
+            }
+
+            if ((flags & Installer.FLAG_CE_STORAGE) != 0) {
+                // Create a native library symlink only if we have native libraries
+                // and if the native libraries are 32 bit libraries. We do not provide
+                // this symlink for 64 bit libraries.
+                if (app.primaryCpuAbi != null && !VMRuntime.is64BitAbi(app.primaryCpuAbi)) {
+                    final String nativeLibPath = app.nativeLibraryDir;
+                    try {
+                        mInstaller.linkNativeLibraryDirectory(volumeUuid, packageName,
+                                nativeLibPath, userId);
+                    } catch (InstallerException e) {
+                        Slog.e(TAG, "Failed to link native for " + packageName + ": " + e);
+                    }
                 }
             }
         }
diff --git a/services/core/java/com/android/server/pm/SELinuxMMAC.java b/services/core/java/com/android/server/pm/SELinuxMMAC.java
index 903d12b..aa10c08 100644
--- a/services/core/java/com/android/server/pm/SELinuxMMAC.java
+++ b/services/core/java/com/android/server/pm/SELinuxMMAC.java
@@ -19,20 +19,24 @@
 import android.content.pm.PackageParser;
 import android.content.pm.Signature;
 import android.os.Environment;
+import android.system.ErrnoException;
+import android.system.Os;
+import android.system.OsConstants;
 import android.util.Slog;
 import android.util.Xml;
 
 import libcore.io.IoUtils;
 
+import org.xmlpull.v1.XmlPullParser;
+import org.xmlpull.v1.XmlPullParserException;
+
 import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
-
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -41,9 +45,6 @@
 import java.util.Map;
 import java.util.Set;
 
-import org.xmlpull.v1.XmlPullParser;
-import org.xmlpull.v1.XmlPullParserException;
-
 /**
  * Centralized access to SELinux MMAC (middleware MAC) implementation. This
  * class is responsible for loading the appropriate mac_permissions.xml file
@@ -63,42 +64,21 @@
     // to synchronize access during policy load and access attempts.
     private static List<Policy> sPolicies = new ArrayList<>();
 
-    // Data policy override version file.
-    private static final String DATA_VERSION_FILE =
-            Environment.getDataDirectory() + "/security/current/selinux_version";
+    /** Path to version on rootfs */
+    private static final File VERSION_FILE = new File("/selinux_version");
 
-    // Base policy version file.
-    private static final String BASE_VERSION_FILE = "/selinux_version";
+    /** Path to MAC permissions on system image */
+    private static final File MAC_PERMISSIONS = new File(Environment.getRootDirectory(),
+            "/etc/security/mac_permissions.xml");
 
-    // Whether override security policies should be loaded.
-    private static final boolean USE_OVERRIDE_POLICY = useOverridePolicy();
+    /** Path to app contexts on rootfs */
+    private static final File SEAPP_CONTEXTS = new File("/seapp_contexts");
 
-    // Data override mac_permissions.xml policy file.
-    private static final String DATA_MAC_PERMISSIONS =
-            Environment.getDataDirectory() + "/security/current/mac_permissions.xml";
+    /** Calculated hash of {@link #SEAPP_CONTEXTS} */
+    private static final byte[] SEAPP_CONTEXTS_HASH = returnHash(SEAPP_CONTEXTS);
 
-    // Base mac_permissions.xml policy file.
-    private static final String BASE_MAC_PERMISSIONS =
-            Environment.getRootDirectory() + "/etc/security/mac_permissions.xml";
-
-    // Determine which mac_permissions.xml file to use.
-    private static final String MAC_PERMISSIONS = USE_OVERRIDE_POLICY ?
-            DATA_MAC_PERMISSIONS : BASE_MAC_PERMISSIONS;
-
-    // Data override seapp_contexts policy file.
-    private static final String DATA_SEAPP_CONTEXTS =
-            Environment.getDataDirectory() + "/security/current/seapp_contexts";
-
-    // Base seapp_contexts policy file.
-    private static final String BASE_SEAPP_CONTEXTS = "/seapp_contexts";
-
-    // Determine which seapp_contexts file to use.
-    private static final String SEAPP_CONTEXTS = USE_OVERRIDE_POLICY ?
-            DATA_SEAPP_CONTEXTS : BASE_SEAPP_CONTEXTS;
-
-    // Stores the hash of the last used seapp_contexts file.
-    private static final String SEAPP_HASH_FILE =
-            Environment.getDataDirectory().toString() + "/system/seapp_hash";
+    /** Attribute where {@link #SEAPP_CONTEXTS_HASH} is stored */
+    private static final String XATTR_SEAPP_HASH = "user.seapp_hash";
 
     // Append privapp to existing seinfo label
     private static final String PRIVILEGED_APP_STR = ":privapp";
@@ -332,71 +312,42 @@
     }
 
     /**
-     * Determines if a recursive restorecon on /data/data and /data/user is needed.
-     * It does this by comparing the SHA-1 of the seapp_contexts file against the
-     * stored hash at /data/system/seapp_hash.
+     * Determines if a recursive restorecon on the given package data directory
+     * is needed. It does this by comparing the SHA-1 of the seapp_contexts file
+     * against the stored hash in an xattr.
+     * <p>
+     * Note that the xattr isn't in the 'security' namespace, so this should
+     * only be run on directories owned by the system.
      *
      * @return Returns true if the restorecon should occur or false otherwise.
      */
-    public static boolean shouldRestorecon() {
-        // Any error with the seapp_contexts file should be fatal
-        byte[] currentHash = null;
+    public static boolean isRestoreconNeeded(File file) {
         try {
-            currentHash = returnHash(SEAPP_CONTEXTS);
-        } catch (IOException ioe) {
-            Slog.e(TAG, "Error with hashing seapp_contexts.", ioe);
-            return false;
+            final byte[] buf = new byte[20];
+            final int len = Os.getxattr(file.getAbsolutePath(), XATTR_SEAPP_HASH, buf);
+            if ((len == 20) && Arrays.equals(SEAPP_CONTEXTS_HASH, buf)) {
+                return false;
+            }
+        } catch (ErrnoException e) {
+            if (e.errno != OsConstants.ENODATA) {
+                Slog.e(TAG, "Failed to read seapp hash for " + file, e);
+            }
         }
 
-        // Push past any error with the stored hash file
-        byte[] storedHash = null;
-        try {
-            storedHash = IoUtils.readFileAsByteArray(SEAPP_HASH_FILE);
-        } catch (IOException ioe) {
-            Slog.w(TAG, "Error opening " + SEAPP_HASH_FILE + ". Assuming first boot.");
-        }
-
-        return (storedHash == null || !MessageDigest.isEqual(storedHash, currentHash));
+        return true;
     }
 
     /**
-     * Stores the SHA-1 of the seapp_contexts to /data/system/seapp_hash.
+     * Stores the SHA-1 of the seapp_contexts into an xattr.
+     * <p>
+     * Note that the xattr isn't in the 'security' namespace, so this should
+     * only be run on directories owned by the system.
      */
-    public static void setRestoreconDone() {
+    public static void setRestoreconDone(File file) {
         try {
-            final byte[] currentHash = returnHash(SEAPP_CONTEXTS);
-            dumpHash(new File(SEAPP_HASH_FILE), currentHash);
-        } catch (IOException ioe) {
-            Slog.e(TAG, "Error with saving hash to " + SEAPP_HASH_FILE, ioe);
-        }
-    }
-
-    /**
-     * Dump the contents of a byte array to a specified file.
-     *
-     * @param file The file that receives the byte array content.
-     * @param content A byte array that will be written to the specified file.
-     * @throws IOException if any failed I/O operation occured.
-     *         Included is the failure to atomically rename the tmp
-     *         file used in the process.
-     */
-    private static void dumpHash(File file, byte[] content) throws IOException {
-        FileOutputStream fos = null;
-        File tmp = null;
-        try {
-            tmp = File.createTempFile("seapp_hash", ".journal", file.getParentFile());
-            tmp.setReadable(true);
-            fos = new FileOutputStream(tmp);
-            fos.write(content);
-            fos.getFD().sync();
-            if (!tmp.renameTo(file)) {
-                throw new IOException("Failure renaming " + file.getCanonicalPath());
-            }
-        } finally {
-            if (tmp != null) {
-                tmp.delete();
-            }
-            IoUtils.closeQuietly(fos);
+            Os.setxattr(file.getAbsolutePath(), XATTR_SEAPP_HASH, SEAPP_CONTEXTS_HASH, 0);
+        } catch (ErrnoException e) {
+            Slog.e(TAG, "Failed to persist seapp hash in " + file, e);
         }
     }
 
@@ -405,33 +356,15 @@
      *
      * @param file The path to the file given as a string.
      * @return Returns the SHA-1 of the file as a byte array.
-     * @throws IOException if any failed I/O operations occured.
      */
-    private static byte[] returnHash(String file) throws IOException {
+    private static byte[] returnHash(File file) {
         try {
-            final byte[] contents = IoUtils.readFileAsByteArray(file);
+            final byte[] contents = IoUtils.readFileAsByteArray(file.getAbsolutePath());
             return MessageDigest.getInstance("SHA-1").digest(contents);
-        } catch (NoSuchAlgorithmException nsae) {
-            throw new RuntimeException(nsae);  // impossible
+        } catch (IOException | NoSuchAlgorithmException e) {
+            throw new RuntimeException(e);
         }
     }
-
-    private static boolean useOverridePolicy() {
-        try {
-            final String overrideVersion = IoUtils.readFileAsString(DATA_VERSION_FILE);
-            final String baseVersion = IoUtils.readFileAsString(BASE_VERSION_FILE);
-            if (overrideVersion.equals(baseVersion)) {
-                return true;
-            }
-            Slog.e(TAG, "Override policy version '" + overrideVersion + "' doesn't match " +
-                   "base version '" + baseVersion + "'. Skipping override policy files.");
-        } catch (FileNotFoundException fnfe) {
-            // Override version file doesn't have to exist so silently ignore.
-        } catch (IOException ioe) {
-            Slog.w(TAG, "Skipping override policy files.", ioe);
-        }
-        return false;
-    }
 }
 
 /**
diff --git a/services/core/java/com/android/server/pm/Settings.java b/services/core/java/com/android/server/pm/Settings.java
index 9fef515..4c976fa 100644
--- a/services/core/java/com/android/server/pm/Settings.java
+++ b/services/core/java/com/android/server/pm/Settings.java
@@ -3893,7 +3893,6 @@
      * given {@link VolumeInfo#fsUuid}.
      */
     List<PackageSetting> getVolumePackagesLPr(String volumeUuid) {
-        Preconditions.checkNotNull(volumeUuid);
         ArrayList<PackageSetting> res = new ArrayList<>();
         for (int i = 0; i < mPackages.size(); i++) {
             final PackageSetting setting = mPackages.valueAt(i);
diff --git a/services/core/java/com/android/server/pm/UserManagerService.java b/services/core/java/com/android/server/pm/UserManagerService.java
index ce6b369..5100faf 100644
--- a/services/core/java/com/android/server/pm/UserManagerService.java
+++ b/services/core/java/com/android/server/pm/UserManagerService.java
@@ -75,6 +75,7 @@
 import com.android.internal.util.XmlUtils;
 import com.android.internal.widget.LockPatternUtils;
 import com.android.server.LocalServices;
+import com.android.server.pm.Installer.StorageFlags;
 
 import org.xmlpull.v1.XmlPullParser;
 import org.xmlpull.v1.XmlPullParserException;
@@ -2394,15 +2395,28 @@
     }
 
     /**
-     * Called right before a user starts.  This will not be called for the system user.
+     * Called right before a user is started. This gives us a chance to prepare
+     * app storage and apply any user restrictions.
      */
     public void onBeforeStartUser(int userId) {
-        synchronized (mRestrictionsLock) {
-            applyUserRestrictionsLR(userId);
+        mPm.reconcileAppsData(userId, Installer.FLAG_DE_STORAGE);
+
+        if (userId != UserHandle.USER_SYSTEM) {
+            synchronized (mRestrictionsLock) {
+                applyUserRestrictionsLR(userId);
+            }
         }
     }
 
     /**
+     * Called right before a user is unlocked. This gives us a chance to prepare
+     * app storage.
+     */
+    public void onBeforeUnlockUser(int userId) {
+        mPm.reconcileAppsData(userId, Installer.FLAG_CE_STORAGE);
+    }
+
+    /**
      * Make a note of the last started time of a user and do some cleanup.
      * @param userId the user that was just foregrounded
      */