Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2018 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package com.android.server; |
| 18 | |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 19 | import static java.lang.annotation.RetentionPolicy.SOURCE; |
| 20 | |
| 21 | import android.annotation.IntDef; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 22 | import android.annotation.Nullable; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 23 | import android.content.Context; |
| 24 | import android.os.Environment; |
| 25 | import android.os.Handler; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 26 | import android.os.Looper; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 27 | import android.os.SystemClock; |
| 28 | import android.text.TextUtils; |
| 29 | import android.util.ArrayMap; |
| 30 | import android.util.AtomicFile; |
| 31 | import android.util.Log; |
| 32 | import android.util.Slog; |
| 33 | import android.util.Xml; |
| 34 | |
| 35 | import com.android.internal.annotations.GuardedBy; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 36 | import com.android.internal.annotations.VisibleForTesting; |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 37 | import com.android.internal.os.BackgroundThread; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 38 | import com.android.internal.util.FastXmlSerializer; |
| 39 | import com.android.internal.util.XmlUtils; |
| 40 | |
| 41 | import libcore.io.IoUtils; |
| 42 | |
| 43 | import org.xmlpull.v1.XmlPullParser; |
| 44 | import org.xmlpull.v1.XmlPullParserException; |
| 45 | import org.xmlpull.v1.XmlSerializer; |
| 46 | |
| 47 | import java.io.File; |
| 48 | import java.io.FileNotFoundException; |
| 49 | import java.io.FileOutputStream; |
| 50 | import java.io.IOException; |
| 51 | import java.io.InputStream; |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 52 | import java.lang.annotation.Retention; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 53 | import java.nio.charset.StandardCharsets; |
| 54 | import java.util.ArrayList; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 55 | import java.util.Collections; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 56 | import java.util.Iterator; |
| 57 | import java.util.List; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 58 | import java.util.Set; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 59 | |
| 60 | /** |
| 61 | * Monitors the health of packages on the system and notifies interested observers when packages |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 62 | * fail. On failure, the registered observer with the least user impacting mitigation will |
| 63 | * be notified. |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 64 | */ |
| 65 | public class PackageWatchdog { |
| 66 | private static final String TAG = "PackageWatchdog"; |
| 67 | // Duration to count package failures before it resets to 0 |
| 68 | private static final int TRIGGER_DURATION_MS = 60000; |
| 69 | // Number of package failures within the duration above before we notify observers |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 70 | static final int TRIGGER_FAILURE_COUNT = 5; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 71 | private static final int DB_VERSION = 1; |
| 72 | private static final String TAG_PACKAGE_WATCHDOG = "package-watchdog"; |
| 73 | private static final String TAG_PACKAGE = "package"; |
| 74 | private static final String TAG_OBSERVER = "observer"; |
| 75 | private static final String ATTR_VERSION = "version"; |
| 76 | private static final String ATTR_NAME = "name"; |
| 77 | private static final String ATTR_DURATION = "duration"; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 78 | |
| 79 | private static PackageWatchdog sPackageWatchdog; |
| 80 | |
| 81 | private final Object mLock = new Object(); |
| 82 | // System server context |
| 83 | private final Context mContext; |
| 84 | // Handler to run package cleanup runnables |
| 85 | private final Handler mTimerHandler; |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 86 | // Handler for processing IO and observer actions |
| 87 | private final Handler mWorkerHandler; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 88 | // Contains (observer-name -> observer-handle) that have ever been registered from |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 89 | // previous boots. Observers with all packages expired are periodically pruned. |
| 90 | // It is saved to disk on system shutdown and repouplated on startup so it survives reboots. |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 91 | @GuardedBy("mLock") |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 92 | private final ArrayMap<String, ObserverInternal> mAllObservers = new ArrayMap<>(); |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 93 | // File containing the XML data of monitored packages /data/system/package-watchdog.xml |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 94 | private final AtomicFile mPolicyFile; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 95 | // Runnable to prune monitored packages that have expired |
| 96 | private final Runnable mPackageCleanup; |
| 97 | // Last SystemClock#uptimeMillis a package clean up was executed. |
| 98 | // 0 if mPackageCleanup not running. |
| 99 | private long mUptimeAtLastRescheduleMs; |
| 100 | // Duration a package cleanup was last scheduled for. |
| 101 | // 0 if mPackageCleanup not running. |
| 102 | private long mDurationAtLastReschedule; |
| 103 | |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 104 | // TODO(zezeozue): Remove redundant context param |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 105 | private PackageWatchdog(Context context) { |
| 106 | mContext = context; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 107 | mPolicyFile = new AtomicFile(new File(new File(Environment.getDataDirectory(), "system"), |
| 108 | "package-watchdog.xml")); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 109 | mTimerHandler = new Handler(Looper.myLooper()); |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 110 | mWorkerHandler = BackgroundThread.getHandler(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 111 | mPackageCleanup = this::rescheduleCleanup; |
| 112 | loadFromFile(); |
| 113 | } |
| 114 | |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 115 | /** |
| 116 | * Creates a PackageWatchdog for testing that uses the same {@code looper} for all handlers |
| 117 | * and creates package-watchdog.xml in an apps data directory. |
| 118 | */ |
| 119 | @VisibleForTesting |
| 120 | PackageWatchdog(Context context, Looper looper) { |
| 121 | mContext = context; |
| 122 | mPolicyFile = new AtomicFile(new File(context.getFilesDir(), "package-watchdog.xml")); |
| 123 | mTimerHandler = new Handler(looper); |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 124 | mWorkerHandler = mTimerHandler; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 125 | mPackageCleanup = this::rescheduleCleanup; |
| 126 | loadFromFile(); |
| 127 | } |
| 128 | |
| 129 | |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 130 | /** Creates or gets singleton instance of PackageWatchdog. */ |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 131 | public static PackageWatchdog getInstance(Context context) { |
| 132 | synchronized (PackageWatchdog.class) { |
| 133 | if (sPackageWatchdog == null) { |
| 134 | sPackageWatchdog = new PackageWatchdog(context); |
| 135 | } |
| 136 | return sPackageWatchdog; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 137 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 138 | } |
| 139 | |
| 140 | /** |
| 141 | * Registers {@code observer} to listen for package failures |
| 142 | * |
| 143 | * <p>Observers are expected to call this on boot. It does not specify any packages but |
| 144 | * it will resume observing any packages requested from a previous boot. |
| 145 | */ |
| 146 | public void registerHealthObserver(PackageHealthObserver observer) { |
| 147 | synchronized (mLock) { |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 148 | ObserverInternal internalObserver = mAllObservers.get(observer.getName()); |
| 149 | if (internalObserver != null) { |
| 150 | internalObserver.mRegisteredObserver = observer; |
| 151 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 152 | if (mDurationAtLastReschedule == 0) { |
| 153 | // Nothing running, schedule |
| 154 | rescheduleCleanup(); |
| 155 | } |
| 156 | } |
| 157 | } |
| 158 | |
| 159 | /** |
| 160 | * Starts observing the health of the {@code packages} for {@code observer} and notifies |
| 161 | * {@code observer} of any package failures within the monitoring duration. |
| 162 | * |
| 163 | * <p>If {@code observer} is already monitoring a package in {@code packageNames}, |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 164 | * the monitoring window of that package will be reset to {@code durationMs}. |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 165 | * |
| 166 | * @throws IllegalArgumentException if {@code packageNames} is empty |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 167 | * or {@code durationMs} is less than 1 |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 168 | */ |
| 169 | public void startObservingHealth(PackageHealthObserver observer, List<String> packageNames, |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 170 | long durationMs) { |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 171 | if (packageNames.isEmpty() || durationMs < 1) { |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 172 | throw new IllegalArgumentException("Observation not started, no packages specified" |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 173 | + "or invalid duration"); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 174 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 175 | List<MonitoredPackage> packages = new ArrayList<>(); |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 176 | for (int i = 0; i < packageNames.size(); i++) { |
| 177 | packages.add(new MonitoredPackage(packageNames.get(i), durationMs)); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 178 | } |
| 179 | synchronized (mLock) { |
| 180 | ObserverInternal oldObserver = mAllObservers.get(observer.getName()); |
| 181 | if (oldObserver == null) { |
| 182 | Slog.d(TAG, observer.getName() + " started monitoring health of packages " |
| 183 | + packageNames); |
| 184 | mAllObservers.put(observer.getName(), |
| 185 | new ObserverInternal(observer.getName(), packages)); |
| 186 | } else { |
| 187 | Slog.d(TAG, observer.getName() + " added the following packages to monitor " |
| 188 | + packageNames); |
| 189 | oldObserver.updatePackages(packages); |
| 190 | } |
| 191 | } |
| 192 | registerHealthObserver(observer); |
| 193 | // Always reschedule because we may need to expire packages |
| 194 | // earlier than we are already scheduled for |
| 195 | rescheduleCleanup(); |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 196 | saveToFileAsync(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 197 | } |
| 198 | |
| 199 | /** |
| 200 | * Unregisters {@code observer} from listening to package failure. |
| 201 | * Additionally, this stops observing any packages that may have previously been observed |
| 202 | * even from a previous boot. |
| 203 | */ |
| 204 | public void unregisterHealthObserver(PackageHealthObserver observer) { |
| 205 | synchronized (mLock) { |
| 206 | mAllObservers.remove(observer.getName()); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 207 | } |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 208 | saveToFileAsync(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 209 | } |
| 210 | |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 211 | /** |
| 212 | * Returns packages observed by {@code observer} |
| 213 | * |
| 214 | * @return an empty set if {@code observer} has some packages observerd from a previous boot |
| 215 | * but has not registered itself in the current boot to receive notifications. Returns null |
| 216 | * if there are no active packages monitored from any boot. |
| 217 | */ |
| 218 | @Nullable |
| 219 | public Set<String> getPackages(PackageHealthObserver observer) { |
| 220 | synchronized (mLock) { |
| 221 | for (int i = 0; i < mAllObservers.size(); i++) { |
| 222 | if (observer.getName().equals(mAllObservers.keyAt(i))) { |
| 223 | if (observer.equals(mAllObservers.valueAt(i).mRegisteredObserver)) { |
| 224 | return mAllObservers.valueAt(i).mPackages.keySet(); |
| 225 | } |
| 226 | return Collections.emptySet(); |
| 227 | } |
| 228 | } |
| 229 | } |
| 230 | return null; |
| 231 | } |
| 232 | |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 233 | // TODO(zezeozue:) Accept current versionCodes of failing packages? |
| 234 | /** |
| 235 | * Called when a process fails either due to a crash or ANR. |
| 236 | * |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 237 | * <p>For each package contained in the process, one registered observer with the least user |
| 238 | * impact will be notified for mitigation. |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 239 | * |
| 240 | * <p>This method could be called frequently if there is a severe problem on the device. |
| 241 | */ |
| 242 | public void onPackageFailure(String[] packages) { |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 243 | mWorkerHandler.post(() -> { |
| 244 | synchronized (mLock) { |
| 245 | if (mAllObservers.isEmpty()) { |
| 246 | return; |
| 247 | } |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 248 | |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 249 | for (int pIndex = 0; pIndex < packages.length; pIndex++) { |
| 250 | String packageToReport = packages[pIndex]; |
| 251 | // Observer that will receive failure for packageToReport |
| 252 | PackageHealthObserver currentObserverToNotify = null; |
| 253 | int currentObserverImpact = Integer.MAX_VALUE; |
| 254 | |
| 255 | // Find observer with least user impact |
| 256 | for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { |
| 257 | ObserverInternal observer = mAllObservers.valueAt(oIndex); |
| 258 | PackageHealthObserver registeredObserver = observer.mRegisteredObserver; |
| 259 | if (registeredObserver != null |
| 260 | && observer.onPackageFailure(packageToReport)) { |
| 261 | int impact = registeredObserver.onHealthCheckFailed(packageToReport); |
| 262 | if (impact != PackageHealthObserverImpact.USER_IMPACT_NONE |
| 263 | && impact < currentObserverImpact) { |
| 264 | currentObserverToNotify = registeredObserver; |
| 265 | currentObserverImpact = impact; |
| 266 | } |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | // Execute action with least user impact |
| 271 | if (currentObserverToNotify != null) { |
| 272 | currentObserverToNotify.execute(packageToReport); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 273 | } |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 274 | } |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 275 | } |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 276 | }); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 277 | } |
| 278 | |
| 279 | // TODO(zezeozue): Optimize write? Maybe only write a separate smaller file? |
| 280 | // This currently adds about 7ms extra to shutdown thread |
| 281 | /** Writes the package information to file during shutdown. */ |
| 282 | public void writeNow() { |
| 283 | if (!mAllObservers.isEmpty()) { |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 284 | mWorkerHandler.removeCallbacks(this::saveToFile); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 285 | pruneObservers(SystemClock.uptimeMillis() - mUptimeAtLastRescheduleMs); |
| 286 | saveToFile(); |
| 287 | Slog.i(TAG, "Last write to update package durations"); |
| 288 | } |
| 289 | } |
| 290 | |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 291 | /** Possible severity values of the user impact of a {@link PackageHealthObserver#execute}. */ |
| 292 | @Retention(SOURCE) |
| 293 | @IntDef(value = {PackageHealthObserverImpact.USER_IMPACT_NONE, |
| 294 | PackageHealthObserverImpact.USER_IMPACT_LOW, |
| 295 | PackageHealthObserverImpact.USER_IMPACT_MEDIUM, |
| 296 | PackageHealthObserverImpact.USER_IMPACT_HIGH}) |
| 297 | public @interface PackageHealthObserverImpact { |
| 298 | /** No action to take. */ |
| 299 | int USER_IMPACT_NONE = 0; |
| 300 | /* Action has low user impact, user of a device will barely notice. */ |
| 301 | int USER_IMPACT_LOW = 1; |
| 302 | /* Action has medium user impact, user of a device will likely notice. */ |
| 303 | int USER_IMPACT_MEDIUM = 3; |
| 304 | /* Action has high user impact, a last resort, user of a device will be very frustrated. */ |
| 305 | int USER_IMPACT_HIGH = 5; |
| 306 | } |
| 307 | |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 308 | /** Register instances of this interface to receive notifications on package failure. */ |
| 309 | public interface PackageHealthObserver { |
| 310 | /** |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 311 | * Called when health check fails for the {@code packageName}. |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 312 | * |
| 313 | * @return any one of {@link PackageHealthObserverImpact} to express the impact |
| 314 | * to the user on {@link #execute} |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 315 | */ |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 316 | @PackageHealthObserverImpact int onHealthCheckFailed(String packageName); |
| 317 | |
| 318 | /** |
| 319 | * Executes mitigation for {@link #onHealthCheckFailed}. |
| 320 | * |
| 321 | * @return {@code true} if action was executed successfully, {@code false} otherwise |
| 322 | */ |
| 323 | boolean execute(String packageName); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 324 | |
| 325 | // TODO(zezeozue): Ensure uniqueness? |
| 326 | /** |
| 327 | * Identifier for the observer, should not change across device updates otherwise the |
| 328 | * watchdog may drop observing packages with the old name. |
| 329 | */ |
| 330 | String getName(); |
| 331 | } |
| 332 | |
| 333 | /** Reschedules handler to prune expired packages from observers. */ |
| 334 | private void rescheduleCleanup() { |
| 335 | synchronized (mLock) { |
| 336 | long nextDurationToScheduleMs = getEarliestPackageExpiryLocked(); |
| 337 | if (nextDurationToScheduleMs == Long.MAX_VALUE) { |
| 338 | Slog.i(TAG, "No monitored packages, ending package cleanup"); |
| 339 | mDurationAtLastReschedule = 0; |
| 340 | mUptimeAtLastRescheduleMs = 0; |
| 341 | return; |
| 342 | } |
| 343 | long uptimeMs = SystemClock.uptimeMillis(); |
| 344 | // O if mPackageCleanup not running |
| 345 | long elapsedDurationMs = mUptimeAtLastRescheduleMs == 0 |
| 346 | ? 0 : uptimeMs - mUptimeAtLastRescheduleMs; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 347 | // Less than O if mPackageCleanup unexpectedly didn't run yet even though |
| 348 | // and we are past the last duration scheduled to run |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 349 | long remainingDurationMs = mDurationAtLastReschedule - elapsedDurationMs; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 350 | if (mUptimeAtLastRescheduleMs == 0 |
| 351 | || remainingDurationMs <= 0 |
| 352 | || nextDurationToScheduleMs < remainingDurationMs) { |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 353 | // First schedule or an earlier reschedule |
| 354 | pruneObservers(elapsedDurationMs); |
| 355 | mTimerHandler.removeCallbacks(mPackageCleanup); |
| 356 | mTimerHandler.postDelayed(mPackageCleanup, nextDurationToScheduleMs); |
| 357 | mDurationAtLastReschedule = nextDurationToScheduleMs; |
| 358 | mUptimeAtLastRescheduleMs = uptimeMs; |
| 359 | } |
| 360 | } |
| 361 | } |
| 362 | |
| 363 | /** |
| 364 | * Returns the earliest time a package should expire. |
| 365 | * @returns Long#MAX_VALUE if there are no observed packages. |
| 366 | */ |
| 367 | private long getEarliestPackageExpiryLocked() { |
| 368 | long shortestDurationMs = Long.MAX_VALUE; |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 369 | for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { |
| 370 | ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex).mPackages; |
| 371 | for (int pIndex = 0; pIndex < packages.size(); pIndex++) { |
| 372 | long duration = packages.valueAt(pIndex).mDurationMs; |
| 373 | if (duration < shortestDurationMs) { |
| 374 | shortestDurationMs = duration; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 375 | } |
| 376 | } |
| 377 | } |
| 378 | Slog.v(TAG, "Earliest package time is " + shortestDurationMs); |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 379 | |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 380 | return shortestDurationMs; |
| 381 | } |
| 382 | |
| 383 | /** |
| 384 | * Removes {@code elapsedMs} milliseconds from all durations on monitored packages. |
| 385 | * Discards expired packages and discards observers without any packages. |
| 386 | */ |
| 387 | private void pruneObservers(long elapsedMs) { |
| 388 | if (elapsedMs == 0) { |
| 389 | return; |
| 390 | } |
| 391 | synchronized (mLock) { |
| 392 | Slog.d(TAG, "Removing expired packages after " + elapsedMs + "ms"); |
| 393 | Iterator<ObserverInternal> it = mAllObservers.values().iterator(); |
| 394 | while (it.hasNext()) { |
| 395 | ObserverInternal observer = it.next(); |
| 396 | if (!observer.updateMonitoringDurations(elapsedMs)) { |
| 397 | Slog.i(TAG, "Discarding observer " + observer.mName + ". All packages expired"); |
| 398 | it.remove(); |
| 399 | } |
| 400 | } |
| 401 | } |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 402 | saveToFileAsync(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 403 | } |
| 404 | |
| 405 | /** |
| 406 | * Loads mAllObservers from file. |
| 407 | * |
| 408 | * <p>Note that this is <b>not</b> thread safe and should only called be called |
| 409 | * from the constructor. |
| 410 | */ |
| 411 | private void loadFromFile() { |
| 412 | InputStream infile = null; |
| 413 | mAllObservers.clear(); |
| 414 | try { |
| 415 | infile = mPolicyFile.openRead(); |
| 416 | final XmlPullParser parser = Xml.newPullParser(); |
| 417 | parser.setInput(infile, StandardCharsets.UTF_8.name()); |
| 418 | XmlUtils.beginDocument(parser, TAG_PACKAGE_WATCHDOG); |
| 419 | int outerDepth = parser.getDepth(); |
| 420 | while (XmlUtils.nextElementWithin(parser, outerDepth)) { |
| 421 | ObserverInternal observer = ObserverInternal.read(parser); |
| 422 | if (observer != null) { |
| 423 | mAllObservers.put(observer.mName, observer); |
| 424 | } |
| 425 | } |
| 426 | } catch (FileNotFoundException e) { |
| 427 | // Nothing to monitor |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 428 | } catch (IOException | NumberFormatException | XmlPullParserException e) { |
| 429 | Log.wtf(TAG, "Unable to read monitored packages, deleting file", e); |
| 430 | mPolicyFile.delete(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 431 | } finally { |
| 432 | IoUtils.closeQuietly(infile); |
| 433 | } |
| 434 | } |
| 435 | |
| 436 | /** |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 437 | * Persists mAllObservers to file. Threshold information is ignored. |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 438 | */ |
| 439 | private boolean saveToFile() { |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 440 | synchronized (mLock) { |
| 441 | FileOutputStream stream; |
| 442 | try { |
| 443 | stream = mPolicyFile.startWrite(); |
| 444 | } catch (IOException e) { |
| 445 | Slog.w(TAG, "Cannot update monitored packages", e); |
| 446 | return false; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 447 | } |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 448 | |
| 449 | try { |
| 450 | XmlSerializer out = new FastXmlSerializer(); |
| 451 | out.setOutput(stream, StandardCharsets.UTF_8.name()); |
| 452 | out.startDocument(null, true); |
| 453 | out.startTag(null, TAG_PACKAGE_WATCHDOG); |
| 454 | out.attribute(null, ATTR_VERSION, Integer.toString(DB_VERSION)); |
| 455 | for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { |
| 456 | mAllObservers.valueAt(oIndex).write(out); |
| 457 | } |
| 458 | out.endTag(null, TAG_PACKAGE_WATCHDOG); |
| 459 | out.endDocument(); |
| 460 | mPolicyFile.finishWrite(stream); |
| 461 | return true; |
| 462 | } catch (IOException e) { |
| 463 | Slog.w(TAG, "Failed to save monitored packages, restoring backup", e); |
| 464 | mPolicyFile.failWrite(stream); |
| 465 | return false; |
| 466 | } finally { |
| 467 | IoUtils.closeQuietly(stream); |
| 468 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 469 | } |
| 470 | } |
| 471 | |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 472 | private void saveToFileAsync() { |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 473 | mWorkerHandler.removeCallbacks(this::saveToFile); |
| 474 | mWorkerHandler.post(this::saveToFile); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 475 | } |
| 476 | |
| 477 | /** |
| 478 | * Represents an observer monitoring a set of packages along with the failure thresholds for |
| 479 | * each package. |
| 480 | */ |
| 481 | static class ObserverInternal { |
| 482 | public final String mName; |
| 483 | public final ArrayMap<String, MonitoredPackage> mPackages; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame] | 484 | @Nullable |
| 485 | public PackageHealthObserver mRegisteredObserver; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 486 | |
| 487 | ObserverInternal(String name, List<MonitoredPackage> packages) { |
| 488 | mName = name; |
| 489 | mPackages = new ArrayMap<>(); |
| 490 | updatePackages(packages); |
| 491 | } |
| 492 | |
| 493 | /** |
| 494 | * Writes important details to file. Doesn't persist any package failure thresholds. |
| 495 | * |
| 496 | * <p>Note that this method is <b>not</b> thread safe. It should only be called from |
| 497 | * #saveToFile which runs on a single threaded handler. |
| 498 | */ |
| 499 | public boolean write(XmlSerializer out) { |
| 500 | try { |
| 501 | out.startTag(null, TAG_OBSERVER); |
| 502 | out.attribute(null, ATTR_NAME, mName); |
| 503 | for (int i = 0; i < mPackages.size(); i++) { |
| 504 | MonitoredPackage p = mPackages.valueAt(i); |
| 505 | out.startTag(null, TAG_PACKAGE); |
| 506 | out.attribute(null, ATTR_NAME, p.mName); |
| 507 | out.attribute(null, ATTR_DURATION, String.valueOf(p.mDurationMs)); |
| 508 | out.endTag(null, TAG_PACKAGE); |
| 509 | } |
| 510 | out.endTag(null, TAG_OBSERVER); |
| 511 | return true; |
| 512 | } catch (IOException e) { |
| 513 | Slog.w(TAG, "Cannot save observer", e); |
| 514 | return false; |
| 515 | } |
| 516 | } |
| 517 | |
| 518 | public void updatePackages(List<MonitoredPackage> packages) { |
| 519 | synchronized (mName) { |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 520 | for (int pIndex = 0; pIndex < packages.size(); pIndex++) { |
| 521 | MonitoredPackage p = packages.get(pIndex); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 522 | mPackages.put(p.mName, p); |
| 523 | } |
| 524 | } |
| 525 | } |
| 526 | |
| 527 | /** |
| 528 | * Reduces the monitoring durations of all packages observed by this observer by |
| 529 | * {@code elapsedMs}. If any duration is less than 0, the package is removed from |
| 530 | * observation. |
| 531 | * |
| 532 | * @returns {@code true} if there are still packages to be observed, {@code false} otherwise |
| 533 | */ |
| 534 | public boolean updateMonitoringDurations(long elapsedMs) { |
| 535 | List<MonitoredPackage> packages = new ArrayList<>(); |
| 536 | synchronized (mName) { |
| 537 | Iterator<MonitoredPackage> it = mPackages.values().iterator(); |
| 538 | while (it.hasNext()) { |
| 539 | MonitoredPackage p = it.next(); |
| 540 | long newDuration = p.mDurationMs - elapsedMs; |
| 541 | if (newDuration > 0) { |
| 542 | p.mDurationMs = newDuration; |
| 543 | } else { |
| 544 | it.remove(); |
| 545 | } |
| 546 | } |
| 547 | return !mPackages.isEmpty(); |
| 548 | } |
| 549 | } |
| 550 | |
| 551 | /** |
| 552 | * Increments failure counts of {@code packageName}. |
| 553 | * @returns {@code true} if failure threshold is exceeded, {@code false} otherwise |
| 554 | */ |
| 555 | public boolean onPackageFailure(String packageName) { |
| 556 | synchronized (mName) { |
| 557 | MonitoredPackage p = mPackages.get(packageName); |
| 558 | if (p != null) { |
| 559 | return p.onFailure(); |
| 560 | } |
| 561 | return false; |
| 562 | } |
| 563 | } |
| 564 | |
| 565 | /** |
| 566 | * Returns one ObserverInternal from the {@code parser} and advances its state. |
| 567 | * |
| 568 | * <p>Note that this method is <b>not</b> thread safe. It should only be called from |
| 569 | * #loadFromFile which in turn is only called on construction of the |
| 570 | * singleton PackageWatchdog. |
| 571 | **/ |
| 572 | public static ObserverInternal read(XmlPullParser parser) { |
| 573 | String observerName = null; |
| 574 | if (TAG_OBSERVER.equals(parser.getName())) { |
| 575 | observerName = parser.getAttributeValue(null, ATTR_NAME); |
| 576 | if (TextUtils.isEmpty(observerName)) { |
| 577 | return null; |
| 578 | } |
| 579 | } |
| 580 | List<MonitoredPackage> packages = new ArrayList<>(); |
| 581 | int innerDepth = parser.getDepth(); |
| 582 | try { |
| 583 | while (XmlUtils.nextElementWithin(parser, innerDepth)) { |
| 584 | if (TAG_PACKAGE.equals(parser.getName())) { |
| 585 | String packageName = parser.getAttributeValue(null, ATTR_NAME); |
| 586 | long duration = Long.parseLong( |
| 587 | parser.getAttributeValue(null, ATTR_DURATION)); |
| 588 | if (!TextUtils.isEmpty(packageName)) { |
| 589 | packages.add(new MonitoredPackage(packageName, duration)); |
| 590 | } |
| 591 | } |
| 592 | } |
| 593 | } catch (IOException e) { |
| 594 | return null; |
| 595 | } catch (XmlPullParserException e) { |
| 596 | return null; |
| 597 | } |
| 598 | if (packages.isEmpty()) { |
| 599 | return null; |
| 600 | } |
| 601 | return new ObserverInternal(observerName, packages); |
| 602 | } |
| 603 | } |
| 604 | |
| 605 | /** Represents a package along with the time it should be monitored for. */ |
| 606 | static class MonitoredPackage { |
| 607 | public final String mName; |
| 608 | // System uptime duration to monitor package |
| 609 | public long mDurationMs; |
| 610 | // System uptime of first package failure |
| 611 | private long mUptimeStartMs; |
| 612 | // Number of failures since mUptimeStartMs |
| 613 | private int mFailures; |
| 614 | |
| 615 | MonitoredPackage(String name, long durationMs) { |
| 616 | mName = name; |
| 617 | mDurationMs = durationMs; |
| 618 | } |
| 619 | |
| 620 | /** |
| 621 | * Increment package failures or resets failure count depending on the last package failure. |
| 622 | * |
| 623 | * @return {@code true} if failure count exceeds a threshold, {@code false} otherwise |
| 624 | */ |
| 625 | public synchronized boolean onFailure() { |
| 626 | final long now = SystemClock.uptimeMillis(); |
| 627 | final long duration = now - mUptimeStartMs; |
| 628 | if (duration > TRIGGER_DURATION_MS) { |
| 629 | // TODO(zezeozue): Reseting to 1 is not correct |
| 630 | // because there may be more than 1 failure in the last trigger window from now |
| 631 | // This is the RescueParty impl, will leave for now |
| 632 | mFailures = 1; |
| 633 | mUptimeStartMs = now; |
| 634 | } else { |
| 635 | mFailures++; |
| 636 | } |
Zimuzo | e5009cd | 2019-01-23 18:11:58 +0000 | [diff] [blame] | 637 | boolean failed = mFailures >= TRIGGER_FAILURE_COUNT; |
| 638 | if (failed) { |
| 639 | mFailures = 0; |
| 640 | } |
| 641 | return failed; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 642 | } |
| 643 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 644 | } |