Improve watchdog error reporting.
We now keep track of all the threads that are stopped, and
print stacks for all of them. Also more threads are now adding
themselves to the watchdog.
Unfortunately the stack we get from threads is far less useful
than the stacks from the ANR report, because these don't include
any information about the lock the thread is blocked on and what
thread is holding that lock. For example, here is a test of the
log output from causing a hang in the system process:
W/Watchdog( 5205): *** WATCHDOG KILLING SYSTEM PROCESS: com.android.server.am.ActivityManagerService, main thread
W/Watchdog( 5205): foreground thread stack trace:
W/Watchdog( 5205): at com.android.server.am.ActivityManagerService.monitor(ActivityManagerService.java:14333)
W/Watchdog( 5205): at com.android.server.Watchdog$HandlerChecker.run(Watchdog.java:142)
W/Watchdog( 5205): at android.os.Handler.handleCallback(Handler.java:730)
W/Watchdog( 5205): at android.os.Handler.dispatchMessage(Handler.java:92)
W/Watchdog( 5205): at android.os.Looper.loop(Looper.java:137)
W/Watchdog( 5205): at android.os.HandlerThread.run(HandlerThread.java:61)
W/Watchdog( 5205): main thread stack trace:
W/Watchdog( 5205): at com.android.server.am.ActivityManagerService.broadcastIntent(ActivityManagerService.java:12252)
W/Watchdog( 5205): at android.app.ContextImpl.sendBroadcastAsUser(ContextImpl.java:1158)
W/Watchdog( 5205): at com.android.server.DropBoxManagerService$3.handleMessage(DropBoxManagerService.java:161)
W/Watchdog( 5205): at android.os.Handler.dispatchMessage(Handler.java:99)
W/Watchdog( 5205): at android.os.Looper.loop(Looper.java:137)
W/Watchdog( 5205): at com.android.server.ServerThread.initAndLoop(SystemServer.java:1050)
W/Watchdog( 5205): at com.android.server.SystemServer.init2(SystemServer.java:1125)
W/Watchdog( 5205): at com.android.server.SystemServer.init1(Native Method)
W/Watchdog( 5205): at com.android.server.SystemServer.main(SystemServer.java:1116)
W/Watchdog( 5205): at java.lang.reflect.Method.invokeNative(Native Method)
W/Watchdog( 5205): at java.lang.reflect.Method.invoke(Method.java:525)
W/Watchdog( 5205): at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:774)
W/Watchdog( 5205): at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:590)
W/Watchdog( 5205): at dalvik.system.NativeStart.main(Native Method)
I/Process ( 5205): Sending signal. PID: 5205 SIG: 9
Change-Id: I8ff9892d8d072d8dc599a73de4bdb75e3b1a6e97
diff --git a/services/java/com/android/server/Watchdog.java b/services/java/com/android/server/Watchdog.java
index 070b8e5..ec573f9 100644
--- a/services/java/com/android/server/Watchdog.java
+++ b/services/java/com/android/server/Watchdog.java
@@ -120,6 +120,14 @@
return mCompleted;
}
+ public Thread getThread() {
+ return mHandler.getLooper().getThread();
+ }
+
+ public String getName() {
+ return mName;
+ }
+
public String describeBlockedStateLocked() {
return mCurrentMonitor == null ? mName : mCurrentMonitor.getClass().getName();
}
@@ -256,17 +264,25 @@
return true;
}
- private String describeBlockedCheckersLocked() {
- StringBuilder builder = new StringBuilder(128);
+ private ArrayList<HandlerChecker> getBlockedCheckersLocked() {
+ ArrayList<HandlerChecker> checkers = new ArrayList<HandlerChecker>();
for (int i=0; i<mHandlerCheckers.size(); i++) {
HandlerChecker hc = mHandlerCheckers.get(i);
if (!hc.isCompletedLocked()) {
- if (builder.length() > 0) {
- builder.append(", ");
- }
- builder.append(hc.describeBlockedStateLocked());
+ checkers.add(hc);
}
}
+ return checkers;
+ }
+
+ private String describeCheckersLocked(ArrayList<HandlerChecker> checkers) {
+ StringBuilder builder = new StringBuilder(128);
+ for (int i=0; i<checkers.size(); i++) {
+ if (builder.length() > 0) {
+ builder.append(", ");
+ }
+ builder.append(checkers.get(i).describeBlockedStateLocked());
+ }
return builder.toString();
}
@@ -274,6 +290,7 @@
public void run() {
boolean waitedHalf = false;
while (true) {
+ final ArrayList<HandlerChecker> blockedCheckers;
final String name;
final boolean allowRestart;
synchronized (this) {
@@ -318,7 +335,8 @@
continue;
}
- name = describeBlockedCheckersLocked();
+ blockedCheckers = getBlockedCheckersLocked();
+ name = describeCheckersLocked(blockedCheckers);
allowRestart = mAllowRestart;
}
@@ -395,12 +413,15 @@
Slog.w(TAG, "Restart not allowed: Watchdog is *not* killing the system process");
} else {
Slog.w(TAG, "*** WATCHDOG KILLING SYSTEM PROCESS: " + name);
- Slog.w(TAG, "Main thread stack trace:");
- StackTraceElement[] stackTrace = Looper.getMainLooper().getThread().getStackTrace();
- for (StackTraceElement element: stackTrace) {
- Slog.w(TAG, "\tat " + element);
+ for (int i=0; i<blockedCheckers.size(); i++) {
+ Slog.w(TAG, blockedCheckers.get(i).getName() + " stack trace:");
+ StackTraceElement[] stackTrace
+ = blockedCheckers.get(i).getThread().getStackTrace();
+ for (StackTraceElement element: stackTrace) {
+ Slog.w(TAG, " at " + element);
+ }
}
- Slog.w(TAG, "<End of main thread stack trace>");
+ Slog.w(TAG, "*** GOODBYE!");
Process.killProcess(Process.myPid());
System.exit(10);
}