Improve watchdog diagnostics.
Capture stack traces from the system process using the same
mechanism as ANRs (which will initialize traces.txt, etc).
Also record the watchdog reset in the dropbox for uploading.
Bug: 2475557
diff --git a/services/java/com/android/server/Watchdog.java b/services/java/com/android/server/Watchdog.java
index ee69715..f97f50a 100644
--- a/services/java/com/android/server/Watchdog.java
+++ b/services/java/com/android/server/Watchdog.java
@@ -34,8 +34,10 @@
import android.provider.Settings;
import android.util.Config;
import android.util.EventLog;
+import android.util.Log;
import android.util.Slog;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
@@ -804,19 +806,14 @@
// to timeout on is asleep as well and won't have a chance to run. Causing a false
// positive on when to kill things.
long start = SystemClock.uptimeMillis();
- do {
+ while (timeout > 0 && !mForceKillSystem) {
try {
- wait(timeout);
+ wait(timeout); // notifyAll() is called when mForceKillSystem is set
} catch (InterruptedException e) {
- if (SystemProperties.getBoolean("ro.secure", false)) {
- // If this is a secure build, just log the error.
- Slog.e("WatchDog", "Woof! Woof! Interrupter!");
- } else {
- throw new AssertionError("Someone interrupted the watchdog");
- }
+ Log.wtf(TAG, e);
}
timeout = TIME_TO_WAIT - (SystemClock.uptimeMillis() - start);
- } while (timeout > 0 && !mForceKillSystem);
+ }
if (mCompleted && !mForceKillSystem) {
// The monitors have returned.
@@ -825,22 +822,24 @@
}
// If we got here, that means that the system is most likely hung.
- // First send a SIGQUIT so that we can see where it was hung. Then
- // kill this process so that the system will restart.
+ // First collect stack traces from all threads of the system process.
+ // Then kill this process so that the system will restart.
+
String name = (mCurrentMonitor != null) ? mCurrentMonitor.getClass().getName() : "null";
EventLog.writeEvent(EventLogTags.WATCHDOG, name);
- Process.sendSignal(Process.myPid(), Process.SIGNAL_QUIT);
- // Wait a bit longer before killing so we can make sure that the stacks are captured.
- try {
- Thread.sleep(10*1000);
- } catch (InterruptedException e) {
- }
+ ArrayList pids = new ArrayList();
+ pids.add(Process.myPid());
+ File stack = ActivityManagerService.dumpStackTraces(pids);
+ mActivity.addErrorToDropBox("watchdog", null, null, null, name, null, stack, null);
// Only kill the process if the debugger is not attached.
if (!Debug.isDebuggerConnected()) {
- Slog.i(TAG, "Watchdog is killing the system process");
+ Slog.w(TAG, "*** WATCHDOG KILLING SYSTEM PROCESS: " + name);
Process.killProcess(Process.myPid());
+ System.exit(10);
+ } else {
+ Slog.w(TAG, "Debugger connected: Watchdog is *not* killing the system process");
}
}
}
diff --git a/services/java/com/android/server/am/ActivityManagerService.java b/services/java/com/android/server/am/ActivityManagerService.java
index a263b23..9018872 100644
--- a/services/java/com/android/server/am/ActivityManagerService.java
+++ b/services/java/com/android/server/am/ActivityManagerService.java
@@ -4662,7 +4662,7 @@
* @param pids of dalvik VM processes to dump stack traces for
* @return file containing stack traces, or null if no dump file is configured
*/
- private static File dumpStackTraces(ArrayList<Integer> pids) {
+ public static File dumpStackTraces(ArrayList<Integer> pids) {
String tracesPath = SystemProperties.get("dalvik.vm.stack-trace-file", null);
if (tracesPath == null || tracesPath.length() == 0) {
return null;
@@ -8958,10 +8958,13 @@
* @param logFile to include in the report, null if none
* @param crashInfo giving an application stack trace, null if absent
*/
- private void addErrorToDropBox(String eventType,
+ public void addErrorToDropBox(String eventType,
ProcessRecord process, HistoryRecord activity, HistoryRecord parent,
String subject, String report, File logFile,
ApplicationErrorReport.CrashInfo crashInfo) {
+ // NOTE -- this must never acquire the ActivityManagerService lock,
+ // otherwise the watchdog may be prevented from resetting the system.
+
String dropboxTag;
if (process == null || process.pid == MY_PID) {
dropboxTag = "system_server_" + eventType;