Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2018 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package com.android.server; |
| 18 | |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 19 | import android.annotation.Nullable; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 20 | import android.content.Context; |
| 21 | import android.os.Environment; |
| 22 | import android.os.Handler; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 23 | import android.os.Looper; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 24 | import android.os.SystemClock; |
| 25 | import android.text.TextUtils; |
| 26 | import android.util.ArrayMap; |
| 27 | import android.util.AtomicFile; |
| 28 | import android.util.Log; |
| 29 | import android.util.Slog; |
| 30 | import android.util.Xml; |
| 31 | |
| 32 | import com.android.internal.annotations.GuardedBy; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 33 | import com.android.internal.annotations.VisibleForTesting; |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 34 | import com.android.internal.os.BackgroundThread; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 35 | import com.android.internal.util.FastXmlSerializer; |
| 36 | import com.android.internal.util.XmlUtils; |
| 37 | |
| 38 | import libcore.io.IoUtils; |
| 39 | |
| 40 | import org.xmlpull.v1.XmlPullParser; |
| 41 | import org.xmlpull.v1.XmlPullParserException; |
| 42 | import org.xmlpull.v1.XmlSerializer; |
| 43 | |
| 44 | import java.io.File; |
| 45 | import java.io.FileNotFoundException; |
| 46 | import java.io.FileOutputStream; |
| 47 | import java.io.IOException; |
| 48 | import java.io.InputStream; |
| 49 | import java.nio.charset.StandardCharsets; |
| 50 | import java.util.ArrayList; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 51 | import java.util.Collections; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 52 | import java.util.Iterator; |
| 53 | import java.util.List; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 54 | import java.util.Set; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 55 | |
| 56 | /** |
| 57 | * Monitors the health of packages on the system and notifies interested observers when packages |
| 58 | * fail. All registered observers will be notified until an observer takes a mitigation action. |
| 59 | */ |
| 60 | public class PackageWatchdog { |
| 61 | private static final String TAG = "PackageWatchdog"; |
| 62 | // Duration to count package failures before it resets to 0 |
| 63 | private static final int TRIGGER_DURATION_MS = 60000; |
| 64 | // Number of package failures within the duration above before we notify observers |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 65 | static final int TRIGGER_FAILURE_COUNT = 5; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 66 | private static final int DB_VERSION = 1; |
| 67 | private static final String TAG_PACKAGE_WATCHDOG = "package-watchdog"; |
| 68 | private static final String TAG_PACKAGE = "package"; |
| 69 | private static final String TAG_OBSERVER = "observer"; |
| 70 | private static final String ATTR_VERSION = "version"; |
| 71 | private static final String ATTR_NAME = "name"; |
| 72 | private static final String ATTR_DURATION = "duration"; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 73 | |
| 74 | private static PackageWatchdog sPackageWatchdog; |
| 75 | |
| 76 | private final Object mLock = new Object(); |
| 77 | // System server context |
| 78 | private final Context mContext; |
| 79 | // Handler to run package cleanup runnables |
| 80 | private final Handler mTimerHandler; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 81 | private final Handler mIoHandler; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 82 | // Contains (observer-name -> observer-handle) that have ever been registered from |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 83 | // previous boots. Observers with all packages expired are periodically pruned. |
| 84 | // It is saved to disk on system shutdown and repouplated on startup so it survives reboots. |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 85 | @GuardedBy("mLock") |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 86 | private final ArrayMap<String, ObserverInternal> mAllObservers = new ArrayMap<>(); |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 87 | // File containing the XML data of monitored packages /data/system/package-watchdog.xml |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 88 | private final AtomicFile mPolicyFile; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 89 | // Runnable to prune monitored packages that have expired |
| 90 | private final Runnable mPackageCleanup; |
| 91 | // Last SystemClock#uptimeMillis a package clean up was executed. |
| 92 | // 0 if mPackageCleanup not running. |
| 93 | private long mUptimeAtLastRescheduleMs; |
| 94 | // Duration a package cleanup was last scheduled for. |
| 95 | // 0 if mPackageCleanup not running. |
| 96 | private long mDurationAtLastReschedule; |
| 97 | |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 98 | // TODO(zezeozue): Remove redundant context param |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 99 | private PackageWatchdog(Context context) { |
| 100 | mContext = context; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 101 | mPolicyFile = new AtomicFile(new File(new File(Environment.getDataDirectory(), "system"), |
| 102 | "package-watchdog.xml")); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 103 | mTimerHandler = new Handler(Looper.myLooper()); |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 104 | mIoHandler = BackgroundThread.getHandler(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 105 | mPackageCleanup = this::rescheduleCleanup; |
| 106 | loadFromFile(); |
| 107 | } |
| 108 | |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 109 | /** |
| 110 | * Creates a PackageWatchdog for testing that uses the same {@code looper} for all handlers |
| 111 | * and creates package-watchdog.xml in an apps data directory. |
| 112 | */ |
| 113 | @VisibleForTesting |
| 114 | PackageWatchdog(Context context, Looper looper) { |
| 115 | mContext = context; |
| 116 | mPolicyFile = new AtomicFile(new File(context.getFilesDir(), "package-watchdog.xml")); |
| 117 | mTimerHandler = new Handler(looper); |
| 118 | mIoHandler = mTimerHandler; |
| 119 | mPackageCleanup = this::rescheduleCleanup; |
| 120 | loadFromFile(); |
| 121 | } |
| 122 | |
| 123 | |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 124 | /** Creates or gets singleton instance of PackageWatchdog. */ |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 125 | public static PackageWatchdog getInstance(Context context) { |
| 126 | synchronized (PackageWatchdog.class) { |
| 127 | if (sPackageWatchdog == null) { |
| 128 | sPackageWatchdog = new PackageWatchdog(context); |
| 129 | } |
| 130 | return sPackageWatchdog; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 131 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 132 | } |
| 133 | |
| 134 | /** |
| 135 | * Registers {@code observer} to listen for package failures |
| 136 | * |
| 137 | * <p>Observers are expected to call this on boot. It does not specify any packages but |
| 138 | * it will resume observing any packages requested from a previous boot. |
| 139 | */ |
| 140 | public void registerHealthObserver(PackageHealthObserver observer) { |
| 141 | synchronized (mLock) { |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 142 | ObserverInternal internalObserver = mAllObservers.get(observer.getName()); |
| 143 | if (internalObserver != null) { |
| 144 | internalObserver.mRegisteredObserver = observer; |
| 145 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 146 | if (mDurationAtLastReschedule == 0) { |
| 147 | // Nothing running, schedule |
| 148 | rescheduleCleanup(); |
| 149 | } |
| 150 | } |
| 151 | } |
| 152 | |
| 153 | /** |
| 154 | * Starts observing the health of the {@code packages} for {@code observer} and notifies |
| 155 | * {@code observer} of any package failures within the monitoring duration. |
| 156 | * |
| 157 | * <p>If {@code observer} is already monitoring a package in {@code packageNames}, |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 158 | * the monitoring window of that package will be reset to {@code durationMs}. |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 159 | * |
| 160 | * @throws IllegalArgumentException if {@code packageNames} is empty |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 161 | * or {@code durationMs} is less than 1 |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 162 | */ |
| 163 | public void startObservingHealth(PackageHealthObserver observer, List<String> packageNames, |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 164 | long durationMs) { |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 165 | if (packageNames.isEmpty() || durationMs < 1) { |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 166 | throw new IllegalArgumentException("Observation not started, no packages specified" |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 167 | + "or invalid duration"); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 168 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 169 | List<MonitoredPackage> packages = new ArrayList<>(); |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 170 | for (int i = 0; i < packageNames.size(); i++) { |
| 171 | packages.add(new MonitoredPackage(packageNames.get(i), durationMs)); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 172 | } |
| 173 | synchronized (mLock) { |
| 174 | ObserverInternal oldObserver = mAllObservers.get(observer.getName()); |
| 175 | if (oldObserver == null) { |
| 176 | Slog.d(TAG, observer.getName() + " started monitoring health of packages " |
| 177 | + packageNames); |
| 178 | mAllObservers.put(observer.getName(), |
| 179 | new ObserverInternal(observer.getName(), packages)); |
| 180 | } else { |
| 181 | Slog.d(TAG, observer.getName() + " added the following packages to monitor " |
| 182 | + packageNames); |
| 183 | oldObserver.updatePackages(packages); |
| 184 | } |
| 185 | } |
| 186 | registerHealthObserver(observer); |
| 187 | // Always reschedule because we may need to expire packages |
| 188 | // earlier than we are already scheduled for |
| 189 | rescheduleCleanup(); |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 190 | saveToFileAsync(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 191 | } |
| 192 | |
| 193 | /** |
| 194 | * Unregisters {@code observer} from listening to package failure. |
| 195 | * Additionally, this stops observing any packages that may have previously been observed |
| 196 | * even from a previous boot. |
| 197 | */ |
| 198 | public void unregisterHealthObserver(PackageHealthObserver observer) { |
| 199 | synchronized (mLock) { |
| 200 | mAllObservers.remove(observer.getName()); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 201 | } |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 202 | saveToFileAsync(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 203 | } |
| 204 | |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 205 | /** |
| 206 | * Returns packages observed by {@code observer} |
| 207 | * |
| 208 | * @return an empty set if {@code observer} has some packages observerd from a previous boot |
| 209 | * but has not registered itself in the current boot to receive notifications. Returns null |
| 210 | * if there are no active packages monitored from any boot. |
| 211 | */ |
| 212 | @Nullable |
| 213 | public Set<String> getPackages(PackageHealthObserver observer) { |
| 214 | synchronized (mLock) { |
| 215 | for (int i = 0; i < mAllObservers.size(); i++) { |
| 216 | if (observer.getName().equals(mAllObservers.keyAt(i))) { |
| 217 | if (observer.equals(mAllObservers.valueAt(i).mRegisteredObserver)) { |
| 218 | return mAllObservers.valueAt(i).mPackages.keySet(); |
| 219 | } |
| 220 | return Collections.emptySet(); |
| 221 | } |
| 222 | } |
| 223 | } |
| 224 | return null; |
| 225 | } |
| 226 | |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 227 | // TODO(zezeozue:) Accept current versionCodes of failing packages? |
| 228 | /** |
| 229 | * Called when a process fails either due to a crash or ANR. |
| 230 | * |
| 231 | * <p>All registered observers for the packages contained in the process will be notified in |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 232 | * order of priority until an observer signifies that it has taken action and other observers |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 233 | * should not notified. |
| 234 | * |
| 235 | * <p>This method could be called frequently if there is a severe problem on the device. |
| 236 | */ |
| 237 | public void onPackageFailure(String[] packages) { |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 238 | ArrayMap<String, List<PackageHealthObserver>> packagesToReport = new ArrayMap<>(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 239 | synchronized (mLock) { |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 240 | if (mAllObservers.isEmpty()) { |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 241 | return; |
| 242 | } |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 243 | |
| 244 | for (int pIndex = 0; pIndex < packages.length; pIndex++) { |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 245 | // Observers interested in receiving packageName failures |
| 246 | List<PackageHealthObserver> observersToNotify = new ArrayList<>(); |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 247 | for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 248 | PackageHealthObserver registeredObserver = |
| 249 | mAllObservers.valueAt(oIndex).mRegisteredObserver; |
| 250 | if (registeredObserver != null) { |
| 251 | observersToNotify.add(registeredObserver); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 252 | } |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 253 | } |
| 254 | // Save interested observers and notify them outside the lock |
| 255 | if (!observersToNotify.isEmpty()) { |
| 256 | packagesToReport.put(packages[pIndex], observersToNotify); |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 257 | } |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // Notify observers |
| 262 | for (int pIndex = 0; pIndex < packagesToReport.size(); pIndex++) { |
| 263 | List<PackageHealthObserver> observers = packagesToReport.valueAt(pIndex); |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 264 | String packageName = packages[pIndex]; |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 265 | for (int oIndex = 0; oIndex < observers.size(); oIndex++) { |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 266 | PackageHealthObserver observer = observers.get(oIndex); |
| 267 | if (mAllObservers.get(observer.getName()).onPackageFailure(packageName) |
| 268 | && observer.onHealthCheckFailed(packageName)) { |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 269 | // Observer has handled, do not notify others |
| 270 | break; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 271 | } |
| 272 | } |
| 273 | } |
| 274 | } |
| 275 | |
| 276 | // TODO(zezeozue): Optimize write? Maybe only write a separate smaller file? |
| 277 | // This currently adds about 7ms extra to shutdown thread |
| 278 | /** Writes the package information to file during shutdown. */ |
| 279 | public void writeNow() { |
| 280 | if (!mAllObservers.isEmpty()) { |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 281 | mIoHandler.removeCallbacks(this::saveToFile); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 282 | pruneObservers(SystemClock.uptimeMillis() - mUptimeAtLastRescheduleMs); |
| 283 | saveToFile(); |
| 284 | Slog.i(TAG, "Last write to update package durations"); |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | /** Register instances of this interface to receive notifications on package failure. */ |
| 289 | public interface PackageHealthObserver { |
| 290 | /** |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 291 | * Called when health check fails for the {@code packageName}. |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 292 | * @return {@code true} if action was taken and other observers should not be notified of |
| 293 | * this failure, {@code false} otherwise. |
| 294 | */ |
| 295 | boolean onHealthCheckFailed(String packageName); |
| 296 | |
| 297 | // TODO(zezeozue): Ensure uniqueness? |
| 298 | /** |
| 299 | * Identifier for the observer, should not change across device updates otherwise the |
| 300 | * watchdog may drop observing packages with the old name. |
| 301 | */ |
| 302 | String getName(); |
| 303 | } |
| 304 | |
| 305 | /** Reschedules handler to prune expired packages from observers. */ |
| 306 | private void rescheduleCleanup() { |
| 307 | synchronized (mLock) { |
| 308 | long nextDurationToScheduleMs = getEarliestPackageExpiryLocked(); |
| 309 | if (nextDurationToScheduleMs == Long.MAX_VALUE) { |
| 310 | Slog.i(TAG, "No monitored packages, ending package cleanup"); |
| 311 | mDurationAtLastReschedule = 0; |
| 312 | mUptimeAtLastRescheduleMs = 0; |
| 313 | return; |
| 314 | } |
| 315 | long uptimeMs = SystemClock.uptimeMillis(); |
| 316 | // O if mPackageCleanup not running |
| 317 | long elapsedDurationMs = mUptimeAtLastRescheduleMs == 0 |
| 318 | ? 0 : uptimeMs - mUptimeAtLastRescheduleMs; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 319 | // Less than O if mPackageCleanup unexpectedly didn't run yet even though |
| 320 | // and we are past the last duration scheduled to run |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 321 | long remainingDurationMs = mDurationAtLastReschedule - elapsedDurationMs; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 322 | if (mUptimeAtLastRescheduleMs == 0 |
| 323 | || remainingDurationMs <= 0 |
| 324 | || nextDurationToScheduleMs < remainingDurationMs) { |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 325 | // First schedule or an earlier reschedule |
| 326 | pruneObservers(elapsedDurationMs); |
| 327 | mTimerHandler.removeCallbacks(mPackageCleanup); |
| 328 | mTimerHandler.postDelayed(mPackageCleanup, nextDurationToScheduleMs); |
| 329 | mDurationAtLastReschedule = nextDurationToScheduleMs; |
| 330 | mUptimeAtLastRescheduleMs = uptimeMs; |
| 331 | } |
| 332 | } |
| 333 | } |
| 334 | |
| 335 | /** |
| 336 | * Returns the earliest time a package should expire. |
| 337 | * @returns Long#MAX_VALUE if there are no observed packages. |
| 338 | */ |
| 339 | private long getEarliestPackageExpiryLocked() { |
| 340 | long shortestDurationMs = Long.MAX_VALUE; |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 341 | for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { |
| 342 | ArrayMap<String, MonitoredPackage> packages = mAllObservers.valueAt(oIndex).mPackages; |
| 343 | for (int pIndex = 0; pIndex < packages.size(); pIndex++) { |
| 344 | long duration = packages.valueAt(pIndex).mDurationMs; |
| 345 | if (duration < shortestDurationMs) { |
| 346 | shortestDurationMs = duration; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 347 | } |
| 348 | } |
| 349 | } |
| 350 | Slog.v(TAG, "Earliest package time is " + shortestDurationMs); |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 351 | |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 352 | return shortestDurationMs; |
| 353 | } |
| 354 | |
| 355 | /** |
| 356 | * Removes {@code elapsedMs} milliseconds from all durations on monitored packages. |
| 357 | * Discards expired packages and discards observers without any packages. |
| 358 | */ |
| 359 | private void pruneObservers(long elapsedMs) { |
| 360 | if (elapsedMs == 0) { |
| 361 | return; |
| 362 | } |
| 363 | synchronized (mLock) { |
| 364 | Slog.d(TAG, "Removing expired packages after " + elapsedMs + "ms"); |
| 365 | Iterator<ObserverInternal> it = mAllObservers.values().iterator(); |
| 366 | while (it.hasNext()) { |
| 367 | ObserverInternal observer = it.next(); |
| 368 | if (!observer.updateMonitoringDurations(elapsedMs)) { |
| 369 | Slog.i(TAG, "Discarding observer " + observer.mName + ". All packages expired"); |
| 370 | it.remove(); |
| 371 | } |
| 372 | } |
| 373 | } |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 374 | saveToFileAsync(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 375 | } |
| 376 | |
| 377 | /** |
| 378 | * Loads mAllObservers from file. |
| 379 | * |
| 380 | * <p>Note that this is <b>not</b> thread safe and should only called be called |
| 381 | * from the constructor. |
| 382 | */ |
| 383 | private void loadFromFile() { |
| 384 | InputStream infile = null; |
| 385 | mAllObservers.clear(); |
| 386 | try { |
| 387 | infile = mPolicyFile.openRead(); |
| 388 | final XmlPullParser parser = Xml.newPullParser(); |
| 389 | parser.setInput(infile, StandardCharsets.UTF_8.name()); |
| 390 | XmlUtils.beginDocument(parser, TAG_PACKAGE_WATCHDOG); |
| 391 | int outerDepth = parser.getDepth(); |
| 392 | while (XmlUtils.nextElementWithin(parser, outerDepth)) { |
| 393 | ObserverInternal observer = ObserverInternal.read(parser); |
| 394 | if (observer != null) { |
| 395 | mAllObservers.put(observer.mName, observer); |
| 396 | } |
| 397 | } |
| 398 | } catch (FileNotFoundException e) { |
| 399 | // Nothing to monitor |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 400 | } catch (IOException | NumberFormatException | XmlPullParserException e) { |
| 401 | Log.wtf(TAG, "Unable to read monitored packages, deleting file", e); |
| 402 | mPolicyFile.delete(); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 403 | } finally { |
| 404 | IoUtils.closeQuietly(infile); |
| 405 | } |
| 406 | } |
| 407 | |
| 408 | /** |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 409 | * Persists mAllObservers to file. Threshold information is ignored. |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 410 | */ |
| 411 | private boolean saveToFile() { |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 412 | synchronized (mLock) { |
| 413 | FileOutputStream stream; |
| 414 | try { |
| 415 | stream = mPolicyFile.startWrite(); |
| 416 | } catch (IOException e) { |
| 417 | Slog.w(TAG, "Cannot update monitored packages", e); |
| 418 | return false; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 419 | } |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 420 | |
| 421 | try { |
| 422 | XmlSerializer out = new FastXmlSerializer(); |
| 423 | out.setOutput(stream, StandardCharsets.UTF_8.name()); |
| 424 | out.startDocument(null, true); |
| 425 | out.startTag(null, TAG_PACKAGE_WATCHDOG); |
| 426 | out.attribute(null, ATTR_VERSION, Integer.toString(DB_VERSION)); |
| 427 | for (int oIndex = 0; oIndex < mAllObservers.size(); oIndex++) { |
| 428 | mAllObservers.valueAt(oIndex).write(out); |
| 429 | } |
| 430 | out.endTag(null, TAG_PACKAGE_WATCHDOG); |
| 431 | out.endDocument(); |
| 432 | mPolicyFile.finishWrite(stream); |
| 433 | return true; |
| 434 | } catch (IOException e) { |
| 435 | Slog.w(TAG, "Failed to save monitored packages, restoring backup", e); |
| 436 | mPolicyFile.failWrite(stream); |
| 437 | return false; |
| 438 | } finally { |
| 439 | IoUtils.closeQuietly(stream); |
| 440 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 441 | } |
| 442 | } |
| 443 | |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 444 | private void saveToFileAsync() { |
| 445 | mIoHandler.removeCallbacks(this::saveToFile); |
| 446 | mIoHandler.post(this::saveToFile); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 447 | } |
| 448 | |
| 449 | /** |
| 450 | * Represents an observer monitoring a set of packages along with the failure thresholds for |
| 451 | * each package. |
| 452 | */ |
| 453 | static class ObserverInternal { |
| 454 | public final String mName; |
| 455 | public final ArrayMap<String, MonitoredPackage> mPackages; |
Zimuzo | 3eee438 | 2019-01-08 20:42:39 +0000 | [diff] [blame^] | 456 | @Nullable |
| 457 | public PackageHealthObserver mRegisteredObserver; |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 458 | |
| 459 | ObserverInternal(String name, List<MonitoredPackage> packages) { |
| 460 | mName = name; |
| 461 | mPackages = new ArrayMap<>(); |
| 462 | updatePackages(packages); |
| 463 | } |
| 464 | |
| 465 | /** |
| 466 | * Writes important details to file. Doesn't persist any package failure thresholds. |
| 467 | * |
| 468 | * <p>Note that this method is <b>not</b> thread safe. It should only be called from |
| 469 | * #saveToFile which runs on a single threaded handler. |
| 470 | */ |
| 471 | public boolean write(XmlSerializer out) { |
| 472 | try { |
| 473 | out.startTag(null, TAG_OBSERVER); |
| 474 | out.attribute(null, ATTR_NAME, mName); |
| 475 | for (int i = 0; i < mPackages.size(); i++) { |
| 476 | MonitoredPackage p = mPackages.valueAt(i); |
| 477 | out.startTag(null, TAG_PACKAGE); |
| 478 | out.attribute(null, ATTR_NAME, p.mName); |
| 479 | out.attribute(null, ATTR_DURATION, String.valueOf(p.mDurationMs)); |
| 480 | out.endTag(null, TAG_PACKAGE); |
| 481 | } |
| 482 | out.endTag(null, TAG_OBSERVER); |
| 483 | return true; |
| 484 | } catch (IOException e) { |
| 485 | Slog.w(TAG, "Cannot save observer", e); |
| 486 | return false; |
| 487 | } |
| 488 | } |
| 489 | |
| 490 | public void updatePackages(List<MonitoredPackage> packages) { |
| 491 | synchronized (mName) { |
Zimuzo | cfaed76 | 2019-01-03 21:13:01 +0000 | [diff] [blame] | 492 | for (int pIndex = 0; pIndex < packages.size(); pIndex++) { |
| 493 | MonitoredPackage p = packages.get(pIndex); |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 494 | mPackages.put(p.mName, p); |
| 495 | } |
| 496 | } |
| 497 | } |
| 498 | |
| 499 | /** |
| 500 | * Reduces the monitoring durations of all packages observed by this observer by |
| 501 | * {@code elapsedMs}. If any duration is less than 0, the package is removed from |
| 502 | * observation. |
| 503 | * |
| 504 | * @returns {@code true} if there are still packages to be observed, {@code false} otherwise |
| 505 | */ |
| 506 | public boolean updateMonitoringDurations(long elapsedMs) { |
| 507 | List<MonitoredPackage> packages = new ArrayList<>(); |
| 508 | synchronized (mName) { |
| 509 | Iterator<MonitoredPackage> it = mPackages.values().iterator(); |
| 510 | while (it.hasNext()) { |
| 511 | MonitoredPackage p = it.next(); |
| 512 | long newDuration = p.mDurationMs - elapsedMs; |
| 513 | if (newDuration > 0) { |
| 514 | p.mDurationMs = newDuration; |
| 515 | } else { |
| 516 | it.remove(); |
| 517 | } |
| 518 | } |
| 519 | return !mPackages.isEmpty(); |
| 520 | } |
| 521 | } |
| 522 | |
| 523 | /** |
| 524 | * Increments failure counts of {@code packageName}. |
| 525 | * @returns {@code true} if failure threshold is exceeded, {@code false} otherwise |
| 526 | */ |
| 527 | public boolean onPackageFailure(String packageName) { |
| 528 | synchronized (mName) { |
| 529 | MonitoredPackage p = mPackages.get(packageName); |
| 530 | if (p != null) { |
| 531 | return p.onFailure(); |
| 532 | } |
| 533 | return false; |
| 534 | } |
| 535 | } |
| 536 | |
| 537 | /** |
| 538 | * Returns one ObserverInternal from the {@code parser} and advances its state. |
| 539 | * |
| 540 | * <p>Note that this method is <b>not</b> thread safe. It should only be called from |
| 541 | * #loadFromFile which in turn is only called on construction of the |
| 542 | * singleton PackageWatchdog. |
| 543 | **/ |
| 544 | public static ObserverInternal read(XmlPullParser parser) { |
| 545 | String observerName = null; |
| 546 | if (TAG_OBSERVER.equals(parser.getName())) { |
| 547 | observerName = parser.getAttributeValue(null, ATTR_NAME); |
| 548 | if (TextUtils.isEmpty(observerName)) { |
| 549 | return null; |
| 550 | } |
| 551 | } |
| 552 | List<MonitoredPackage> packages = new ArrayList<>(); |
| 553 | int innerDepth = parser.getDepth(); |
| 554 | try { |
| 555 | while (XmlUtils.nextElementWithin(parser, innerDepth)) { |
| 556 | if (TAG_PACKAGE.equals(parser.getName())) { |
| 557 | String packageName = parser.getAttributeValue(null, ATTR_NAME); |
| 558 | long duration = Long.parseLong( |
| 559 | parser.getAttributeValue(null, ATTR_DURATION)); |
| 560 | if (!TextUtils.isEmpty(packageName)) { |
| 561 | packages.add(new MonitoredPackage(packageName, duration)); |
| 562 | } |
| 563 | } |
| 564 | } |
| 565 | } catch (IOException e) { |
| 566 | return null; |
| 567 | } catch (XmlPullParserException e) { |
| 568 | return null; |
| 569 | } |
| 570 | if (packages.isEmpty()) { |
| 571 | return null; |
| 572 | } |
| 573 | return new ObserverInternal(observerName, packages); |
| 574 | } |
| 575 | } |
| 576 | |
| 577 | /** Represents a package along with the time it should be monitored for. */ |
| 578 | static class MonitoredPackage { |
| 579 | public final String mName; |
| 580 | // System uptime duration to monitor package |
| 581 | public long mDurationMs; |
| 582 | // System uptime of first package failure |
| 583 | private long mUptimeStartMs; |
| 584 | // Number of failures since mUptimeStartMs |
| 585 | private int mFailures; |
| 586 | |
| 587 | MonitoredPackage(String name, long durationMs) { |
| 588 | mName = name; |
| 589 | mDurationMs = durationMs; |
| 590 | } |
| 591 | |
| 592 | /** |
| 593 | * Increment package failures or resets failure count depending on the last package failure. |
| 594 | * |
| 595 | * @return {@code true} if failure count exceeds a threshold, {@code false} otherwise |
| 596 | */ |
| 597 | public synchronized boolean onFailure() { |
| 598 | final long now = SystemClock.uptimeMillis(); |
| 599 | final long duration = now - mUptimeStartMs; |
| 600 | if (duration > TRIGGER_DURATION_MS) { |
| 601 | // TODO(zezeozue): Reseting to 1 is not correct |
| 602 | // because there may be more than 1 failure in the last trigger window from now |
| 603 | // This is the RescueParty impl, will leave for now |
| 604 | mFailures = 1; |
| 605 | mUptimeStartMs = now; |
| 606 | } else { |
| 607 | mFailures++; |
| 608 | } |
| 609 | return mFailures >= TRIGGER_FAILURE_COUNT; |
| 610 | } |
| 611 | } |
Zimuzo | 6efba54 | 2018-11-29 12:47:58 +0000 | [diff] [blame] | 612 | } |