Reconcile apps in 2 phases

During boot app data folders are reconciled in 2 phases:
 - in the constructor only core apps are reconciled. prepareAppData
   for remaining apps is deferred and run on a separate thread (phase 2)
 - Phase 2 must finish before third-party apps can start

Also moved GC to final stages of system server init. GC alone takes ~200 ms.

Overall boot time improvement: ~1 second

Before:
02-17 18:33:33 D/BaseBootTest: successive-boot :
28835.0,29638.0,30205.0,29793.0,29752.0,28228.0,30125.0,28983.0,28487.0,28865.0,
02-17 18:33:33 D/BaseBootTest: successive-boot_avg : 29291.1
02-17 18:33:33 D/BaseBootTest:
SystemServerTiming_StartPackageManagerService :
3150.0,3615.0,3515.0,3495.0,3814.0,3158.0,3746.0,3274.0,3222.0,3607.0,
02-17 18:33:33 D/BaseBootTest:
SystemServerTiming_StartPackageManagerService_avg : 3459.6
02-17 18:33:33 D/BaseBootTest: SystemServerTiming_StartServices :
8244.0,8863.0,9035.0,9832.0,8998.0,8096.0,8719.0,8209.0,8279.0,8754.0,
02-17 18:33:33 D/BaseBootTest: SystemServerTiming_StartServices_avg :
8702.9

After:
02-17 17:59:51 D/BaseBootTest: successive-boot :
27711.0,27607.0,28408.0,28968.0,28397.0,28063.0,27885.0,28483.0,27917.0,29317.0,
02-17 17:59:51 D/BaseBootTest: successive-boot_avg : 28275.6
02-17 17:59:51 D/BaseBootTest:
SystemServerTiming_StartPackageManagerService :
2467.0,2489.0,2369.0,2548.0,2647.0,2523.0,2497.0,2553.0,2482.0,2657.0,
02-17 17:59:51 D/BaseBootTest:
SystemServerTiming_StartPackageManagerService_avg : 2523.2
02-17 17:59:51 D/BaseBootTest: SystemServerTiming_StartServices :
7686.0,7538.0,7598.0,7869.0,7884.0,7950.0,7971.0,8370.0,7696.0,7885.0,
02-17 17:59:51 D/BaseBootTest: SystemServerTiming_StartServices_avg :
7844.7

Test: manual
Bug: 28750609
Change-Id: I3543ef577af1365394775318e40907584ddbe950
diff --git a/services/core/java/com/android/server/pm/PackageManagerService.java b/services/core/java/com/android/server/pm/PackageManagerService.java
index 371a062..3558f3e 100644
--- a/services/core/java/com/android/server/pm/PackageManagerService.java
+++ b/services/core/java/com/android/server/pm/PackageManagerService.java
@@ -251,6 +251,7 @@
 import com.android.internal.os.Zygote;
 import com.android.internal.telephony.CarrierAppUtils;
 import com.android.internal.util.ArrayUtils;
+import com.android.internal.util.ConcurrentUtils;
 import com.android.internal.util.FastPrintWriter;
 import com.android.internal.util.FastXmlSerializer;
 import com.android.internal.util.IndentingPrintWriter;
@@ -265,6 +266,7 @@
 import com.android.server.LocalServices;
 import com.android.server.ServiceThread;
 import com.android.server.SystemConfig;
+import com.android.server.SystemServerInitThreadPool;
 import com.android.server.Watchdog;
 import com.android.server.net.NetworkPolicyManagerInternal;
 import com.android.server.pm.Installer.InstallerException;
@@ -322,6 +324,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -857,6 +860,8 @@
 
     private ArraySet<String> mPrivappPermissionsViolations;
 
+    private Future<?> mPrepareAppDataFuture;
+
     private static class IFVerificationParams {
         PackageParser.Package pkg;
         boolean replacing;
@@ -2757,8 +2762,32 @@
             } else {
                 storageFlags = StorageManager.FLAG_STORAGE_DE | StorageManager.FLAG_STORAGE_CE;
             }
-            reconcileAppsDataLI(StorageManager.UUID_PRIVATE_INTERNAL, UserHandle.USER_SYSTEM,
-                    storageFlags, true /* migrateAppData */);
+            List<String> deferPackages = reconcileAppsDataLI(StorageManager.UUID_PRIVATE_INTERNAL,
+                    UserHandle.USER_SYSTEM, storageFlags, true /* migrateAppData */,
+                    true /* onlyCoreApps */);
+            mPrepareAppDataFuture = SystemServerInitThreadPool.get().submit(() -> {
+                if (deferPackages == null || deferPackages.isEmpty()) {
+                    return;
+                }
+                int count = 0;
+                for (String pkgName : deferPackages) {
+                    PackageParser.Package pkg = null;
+                    synchronized (mPackages) {
+                        PackageSetting ps = mSettings.getPackageLPr(pkgName);
+                        if (ps != null && ps.getInstalled(UserHandle.USER_SYSTEM)) {
+                            pkg = ps.pkg;
+                        }
+                    }
+                    if (pkg != null) {
+                        synchronized (mInstallLock) {
+                            prepareAppDataAndMigrateLIF(pkg, UserHandle.USER_SYSTEM, storageFlags,
+                                    true /* maybeMigrateAppData */);
+                        }
+                        count++;
+                    }
+                }
+                Slog.i(TAG, "Deferred reconcileAppsData finished " + count + " packages");
+            }, "prepareAppData");
 
             // If this is first boot after an OTA, and a normal boot, then
             // we need to clear code cache directories.
@@ -20140,6 +20169,14 @@
         }
     }
 
+    public void waitForAppDataPrepared() {
+        if (mPrepareAppDataFuture == null) {
+            return;
+        }
+        ConcurrentUtils.waitForFutureNoInterrupt(mPrepareAppDataFuture, "wait for prepareAppData");
+        mPrepareAppDataFuture = null;
+    }
+
     @Override
     public boolean isSafeMode() {
         return mSafeMode;
@@ -21590,6 +21627,11 @@
         }
     }
 
+    private void reconcileAppsDataLI(String volumeUuid, int userId, int flags,
+            boolean migrateAppData) {
+        reconcileAppsDataLI(volumeUuid, userId, flags, migrateAppData, false /* onlyCoreApps */);
+    }
+
     /**
      * Reconcile all app data on given mounted volume.
      * <p>
@@ -21598,11 +21640,13 @@
      * <p>
      * Verifies that directories exist and that ownership and labeling is
      * correct for all installed apps.
+     * @returns list of skipped non-core packages (if {@code onlyCoreApps} is true)
      */
-    private void reconcileAppsDataLI(String volumeUuid, int userId, int flags,
-            boolean migrateAppData) {
+    private List<String> reconcileAppsDataLI(String volumeUuid, int userId, int flags,
+            boolean migrateAppData, boolean onlyCoreApps) {
         Slog.v(TAG, "reconcileAppsData for " + volumeUuid + " u" + userId + " 0x"
                 + Integer.toHexString(flags) + " migrateAppData=" + migrateAppData);
+        List<String> result = onlyCoreApps ? new ArrayList<>() : null;
 
         final File ceDir = Environment.getDataUserCeDirectory(volumeUuid, userId);
         final File deDir = Environment.getDataUserDeDirectory(volumeUuid, userId);
@@ -21666,21 +21710,20 @@
                 // and reconcile again once they're scanned
                 continue;
             }
+            // Skip non-core apps if requested
+            if (onlyCoreApps && !ps.pkg.coreApp) {
+                result.add(packageName);
+                continue;
+            }
 
             if (ps.getInstalled(userId)) {
-                prepareAppDataLIF(ps.pkg, userId, flags);
-
-                if (migrateAppData && maybeMigrateAppDataLIF(ps.pkg, userId)) {
-                    // We may have just shuffled around app data directories, so
-                    // prepare them one more time
-                    prepareAppDataLIF(ps.pkg, userId, flags);
-                }
-
+                prepareAppDataAndMigrateLIF(ps.pkg, userId, flags, migrateAppData);
                 preparedCount++;
             }
         }
 
         Slog.v(TAG, "reconcileAppsData finished " + preparedCount + " packages");
+        return result;
     }
 
     /**
@@ -21741,6 +21784,17 @@
         }
     }
 
+    private void prepareAppDataAndMigrateLIF(PackageParser.Package pkg, int userId, int flags,
+            boolean maybeMigrateAppData) {
+        prepareAppDataLIF(pkg, userId, flags);
+
+        if (maybeMigrateAppData && maybeMigrateAppDataLIF(pkg, userId)) {
+            // We may have just shuffled around app data directories, so
+            // prepare them one more time
+            prepareAppDataLIF(pkg, userId, flags);
+        }
+    }
+
     private void prepareAppDataLeafLIF(PackageParser.Package pkg, int userId, int flags) {
         if (DEBUG_APP_DATA) {
             Slog.v(TAG, "prepareAppData for " + pkg.packageName + " u" + userId + " 0x"
diff --git a/services/java/com/android/server/SystemServer.java b/services/java/com/android/server/SystemServer.java
index 7917644..fd5ecd3 100644
--- a/services/java/com/android/server/SystemServer.java
+++ b/services/java/com/android/server/SystemServer.java
@@ -376,6 +376,7 @@
             startBootstrapServices();
             startCoreServices();
             startOtherServices();
+            SystemServerInitThreadPool.shutdown();
         } catch (Throwable ex) {
             Slog.e("System", "******************************************");
             Slog.e("System", "************ Failure starting system services", ex);
@@ -383,7 +384,6 @@
         } finally {
             traceEnd();
         }
-        SystemServerInitThreadPool.shutdown();
 
         // For debug builds, log event loop stalls to dropbox for analysis.
         if (StrictMode.conditionallyEnableDebugLogging()) {
@@ -1694,6 +1694,9 @@
             Watchdog.getInstance().start();
             traceEnd();
 
+            // Wait for all packages to be prepared
+            mPackageManagerService.waitForAppDataPrepared();
+
             // It is now okay to let the various system services start their
             // third party code...
             traceBeginAndSlog("PhaseThirdPartyAppsCanStart");