Remove memory monitoring from the system watchdog
This was originally written as an in-case-we-need-it facility, but was
never actually used in production. It also soaked up a surprising amount
of cpu on occasion, as well as doing sketchy things like demoting the
system_server's primary looper thread to the background cgroup at times.
Change-Id: I9a81a8d1e9caea9e0a1277d97785fe96add438d7
diff --git a/services/java/com/android/server/Watchdog.java b/services/java/com/android/server/Watchdog.java
index d4133f3..a742093 100644
--- a/services/java/com/android/server/Watchdog.java
+++ b/services/java/com/android/server/Watchdog.java
@@ -57,20 +57,10 @@
static final boolean RECORD_KERNEL_THREADS = true;
static final int MONITOR = 2718;
- static final int GLOBAL_PSS = 2719;
static final int TIME_TO_RESTART = DB ? 15*1000 : 60*1000;
static final int TIME_TO_WAIT = TIME_TO_RESTART / 2;
- static final int MEMCHECK_DEFAULT_INTERVAL = DB ? 30 : 30*60; // 30 minutes
- static final int MEMCHECK_DEFAULT_LOG_REALTIME_INTERVAL = DB ? 60 : 2*60*60; // 2 hours
- static final int MEMCHECK_DEFAULT_SYSTEM_SOFT_THRESHOLD = (DB ? 10:16)*1024*1024; // 16MB
- static final int MEMCHECK_DEFAULT_SYSTEM_HARD_THRESHOLD = (DB ? 14:20)*1024*1024; // 20MB
- static final int MEMCHECK_DEFAULT_PHONE_SOFT_THRESHOLD = (DB ? 4:8)*1024*1024; // 8MB
- static final int MEMCHECK_DEFAULT_PHONE_HARD_THRESHOLD = (DB ? 8:12)*1024*1024; // 12MB
-
- static final int MEMCHECK_DEFAULT_EXEC_START_TIME = 1*60*60; // 1:00am
- static final int MEMCHECK_DEFAULT_EXEC_END_TIME = 5*60*60; // 5:00am
static final int MEMCHECK_DEFAULT_MIN_SCREEN_OFF = DB ? 1*60 : 5*60; // 5 minutes
static final int MEMCHECK_DEFAULT_MIN_ALARM = DB ? 1*60 : 3*60; // 3 minutes
static final int MEMCHECK_DEFAULT_RECHECK_INTERVAL = DB ? 1*60 : 5*60; // 5 minutes
@@ -79,14 +69,12 @@
static final int REBOOT_DEFAULT_START_TIME = 3*60*60; // 3:00am
static final int REBOOT_DEFAULT_WINDOW = 60*60; // within 1 hour
- static final String CHECKUP_ACTION = "com.android.service.Watchdog.CHECKUP";
static final String REBOOT_ACTION = "com.android.service.Watchdog.REBOOT";
static Watchdog sWatchdog;
/* This handler will be used to post message back onto the main thread */
final Handler mHandler;
- final Runnable mGlobalPssCollected;
final ArrayList<Monitor> mMonitors = new ArrayList<Monitor>();
ContentResolver mResolver;
BatteryService mBattery;
@@ -97,31 +85,9 @@
boolean mForceKillSystem;
Monitor mCurrentMonitor;
- PssRequestor mPhoneReq;
int mPhonePid;
- int mPhonePss;
-
- long mLastMemCheckTime = -(MEMCHECK_DEFAULT_INTERVAL*1000);
- boolean mHavePss;
- long mLastMemCheckRealtime = -(MEMCHECK_DEFAULT_LOG_REALTIME_INTERVAL*1000);
- boolean mHaveGlobalPss;
- final MemMonitor mSystemMemMonitor = new MemMonitor("system",
- Settings.Secure.MEMCHECK_SYSTEM_ENABLED,
- Settings.Secure.MEMCHECK_SYSTEM_SOFT_THRESHOLD,
- MEMCHECK_DEFAULT_SYSTEM_SOFT_THRESHOLD,
- Settings.Secure.MEMCHECK_SYSTEM_HARD_THRESHOLD,
- MEMCHECK_DEFAULT_SYSTEM_HARD_THRESHOLD);
- final MemMonitor mPhoneMemMonitor = new MemMonitor("com.android.phone",
- Settings.Secure.MEMCHECK_PHONE_ENABLED,
- Settings.Secure.MEMCHECK_PHONE_SOFT_THRESHOLD,
- MEMCHECK_DEFAULT_PHONE_SOFT_THRESHOLD,
- Settings.Secure.MEMCHECK_PHONE_HARD_THRESHOLD,
- MEMCHECK_DEFAULT_PHONE_HARD_THRESHOLD);
final Calendar mCalendar = Calendar.getInstance();
- long mMemcheckLastTime;
- long mMemcheckExecStartTime;
- long mMemcheckExecEndTime;
int mMinScreenOff = MEMCHECK_DEFAULT_MIN_SCREEN_OFF;
int mMinAlarm = MEMCHECK_DEFAULT_MIN_ALARM;
boolean mNeedScheduledCheck;
@@ -140,126 +106,13 @@
int mReqRecheckInterval= -1; // >= 0 if a specific recheck interval has been requested
/**
- * This class monitors the memory in a particular process.
- */
- final class MemMonitor {
- final String mProcessName;
- final String mEnabledSetting;
- final String mSoftSetting;
- final String mHardSetting;
-
- int mSoftThreshold;
- int mHardThreshold;
- boolean mEnabled;
- long mLastPss;
-
- static final int STATE_OK = 0;
- static final int STATE_SOFT = 1;
- static final int STATE_HARD = 2;
- int mState;
-
- MemMonitor(String processName, String enabledSetting,
- String softSetting, int defSoftThreshold,
- String hardSetting, int defHardThreshold) {
- mProcessName = processName;
- mEnabledSetting = enabledSetting;
- mSoftSetting = softSetting;
- mHardSetting = hardSetting;
- mSoftThreshold = defSoftThreshold;
- mHardThreshold = defHardThreshold;
- }
-
- void retrieveSettings(ContentResolver resolver) {
- mSoftThreshold = Settings.Secure.getInt(
- resolver, mSoftSetting, mSoftThreshold);
- mHardThreshold = Settings.Secure.getInt(
- resolver, mHardSetting, mHardThreshold);
- mEnabled = Settings.Secure.getInt(
- resolver, mEnabledSetting, 0) != 0;
- }
-
- boolean checkLocked(long curTime, int pid, int pss) {
- mLastPss = pss;
- if (mLastPss < mSoftThreshold) {
- mState = STATE_OK;
- } else if (mLastPss < mHardThreshold) {
- mState = STATE_SOFT;
- } else {
- mState = STATE_HARD;
- }
- EventLog.writeEvent(EventLogTags.WATCHDOG_PROC_PSS, mProcessName, pid, mLastPss);
-
- if (mState == STATE_OK) {
- // Memory is good, don't recover.
- return false;
- }
-
- if (mState == STATE_HARD) {
- // Memory is really bad, kill right now.
- EventLog.writeEvent(EventLogTags.WATCHDOG_HARD_RESET, mProcessName, pid,
- mHardThreshold, mLastPss);
- return mEnabled;
- }
-
- // It is time to schedule a reset...
- // Check if we are currently within the time to kill processes due
- // to memory use.
- computeMemcheckTimesLocked(curTime);
- String skipReason = null;
- if (curTime < mMemcheckExecStartTime || curTime > mMemcheckExecEndTime) {
- skipReason = "time";
- } else {
- skipReason = shouldWeBeBrutalLocked(curTime);
- }
- EventLog.writeEvent(EventLogTags.WATCHDOG_SOFT_RESET, mProcessName, pid,
- mSoftThreshold, mLastPss, skipReason != null ? skipReason : "");
- if (skipReason != null) {
- mNeedScheduledCheck = true;
- return false;
- }
- return mEnabled;
- }
-
- void clear() {
- mLastPss = 0;
- mState = STATE_OK;
- }
- }
-
- /**
* Used for scheduling monitor callbacks and checking memory usage.
*/
final class HeartbeatHandler extends Handler {
@Override
public void handleMessage(Message msg) {
switch (msg.what) {
- case GLOBAL_PSS: {
- if (mHaveGlobalPss) {
- // During the last pass we collected pss information, so
- // now it is time to report it.
- mHaveGlobalPss = false;
- if (localLOGV) Slog.v(TAG, "Received global pss, logging.");
- logGlobalMemory();
- }
- } break;
-
case MONITOR: {
- if (mHavePss) {
- // During the last pass we collected pss information, so
- // now it is time to report it.
- mHavePss = false;
- if (localLOGV) Slog.v(TAG, "Have pss, checking memory.");
- checkMemory();
- }
-
- if (mHaveGlobalPss) {
- // During the last pass we collected pss information, so
- // now it is time to report it.
- mHaveGlobalPss = false;
- if (localLOGV) Slog.v(TAG, "Have global pss, logging.");
- logGlobalMemory();
- }
-
long now = SystemClock.uptimeMillis();
// See if we should force a reboot.
@@ -274,32 +127,6 @@
checkReboot(false);
}
- // See if we should check memory conditions.
- long memCheckInterval = Settings.Secure.getLong(
- mResolver, Settings.Secure.MEMCHECK_INTERVAL,
- MEMCHECK_DEFAULT_INTERVAL) * 1000;
- if ((mLastMemCheckTime+memCheckInterval) < now) {
- // It is now time to collect pss information. This
- // is async so we won't report it now. And to keep
- // things simple, we will assume that everyone has
- // reported back by the next MONITOR message.
- mLastMemCheckTime = now;
- if (localLOGV) Slog.v(TAG, "Collecting memory usage.");
- collectMemory();
- mHavePss = true;
-
- long memCheckRealtimeInterval = Settings.Secure.getLong(
- mResolver, Settings.Secure.MEMCHECK_LOG_REALTIME_INTERVAL,
- MEMCHECK_DEFAULT_LOG_REALTIME_INTERVAL) * 1000;
- long realtimeNow = SystemClock.elapsedRealtime();
- if ((mLastMemCheckRealtime+memCheckRealtimeInterval) < realtimeNow) {
- mLastMemCheckRealtime = realtimeNow;
- if (localLOGV) Slog.v(TAG, "Collecting global memory usage.");
- collectGlobalMemory();
- mHaveGlobalPss = true;
- }
- }
-
final int size = mMonitors.size();
for (int i = 0 ; i < size ; i++) {
mCurrentMonitor = mMonitors.get(i);
@@ -315,20 +142,6 @@
}
}
- final class GlobalPssCollected implements Runnable {
- public void run() {
- mHandler.sendEmptyMessage(GLOBAL_PSS);
- }
- }
-
- final class CheckupReceiver extends BroadcastReceiver {
- @Override
- public void onReceive(Context c, Intent intent) {
- if (localLOGV) Slog.v(TAG, "Alarm went off, checking memory.");
- checkMemory();
- }
- }
-
final class RebootReceiver extends BroadcastReceiver {
@Override
public void onReceive(Context c, Intent intent) {
@@ -359,27 +172,6 @@
void monitor();
}
- public interface PssRequestor {
- void requestPss();
- }
-
- public class PssStats {
- public int mEmptyPss;
- public int mEmptyCount;
- public int mBackgroundPss;
- public int mBackgroundCount;
- public int mServicePss;
- public int mServiceCount;
- public int mVisiblePss;
- public int mVisibleCount;
- public int mForegroundPss;
- public int mForegroundCount;
-
- public int mNoPssCount;
-
- public int mProcDeaths[] = new int[10];
- }
-
public static Watchdog getInstance() {
if (sWatchdog == null) {
sWatchdog = new Watchdog();
@@ -391,7 +183,6 @@
private Watchdog() {
super("watchdog");
mHandler = new HeartbeatHandler();
- mGlobalPssCollected = new GlobalPssCollected();
}
public void init(Context context, BatteryService battery,
@@ -403,11 +194,6 @@
mAlarm = alarm;
mActivity = activity;
- context.registerReceiver(new CheckupReceiver(),
- new IntentFilter(CHECKUP_ACTION));
- mCheckupIntent = PendingIntent.getBroadcast(context,
- 0, new Intent(CHECKUP_ACTION), 0);
-
context.registerReceiver(new RebootReceiver(),
new IntentFilter(REBOOT_ACTION));
mRebootIntent = PendingIntent.getBroadcast(context,
@@ -420,20 +206,10 @@
mBootTime = System.currentTimeMillis();
}
- public void processStarted(PssRequestor req, String name, int pid) {
+ public void processStarted(String name, int pid) {
synchronized (this) {
if ("com.android.phone".equals(name)) {
- mPhoneReq = req;
mPhonePid = pid;
- mPhonePss = 0;
- }
- }
- }
-
- public void reportPss(PssRequestor req, String name, int pss) {
- synchronized (this) {
- if (mPhoneReq == req) {
- mPhonePss = pss;
}
}
}
@@ -447,152 +223,6 @@
}
}
- /**
- * Retrieve memory usage information from specific processes being
- * monitored. This is an async operation, so must be done before doing
- * memory checks.
- */
- void collectMemory() {
- synchronized (this) {
- if (mPhoneReq != null) {
- mPhoneReq.requestPss();
- }
- }
- }
-
- /**
- * Retrieve memory usage over all application processes. This is an
- * async operation, so must be done before doing memory checks.
- */
- void collectGlobalMemory() {
- mActivity.requestPss(mGlobalPssCollected);
- }
-
- /**
- * Check memory usage in the system, scheduling kills/reboots as needed.
- * This always runs on the mHandler thread.
- */
- void checkMemory() {
- boolean needScheduledCheck;
- long curTime;
- long nextTime = 0;
-
- long recheckInterval = Settings.Secure.getLong(
- mResolver, Settings.Secure.MEMCHECK_RECHECK_INTERVAL,
- MEMCHECK_DEFAULT_RECHECK_INTERVAL) * 1000;
-
- mSystemMemMonitor.retrieveSettings(mResolver);
- mPhoneMemMonitor.retrieveSettings(mResolver);
- retrieveBrutalityAmount();
-
- synchronized (this) {
- curTime = System.currentTimeMillis();
- mNeedScheduledCheck = false;
-
- // How is the system doing?
- if (mSystemMemMonitor.checkLocked(curTime, Process.myPid(),
- (int)Process.getPss(Process.myPid()))) {
- // Not good! Time to suicide.
- mForceKillSystem = true;
- notifyAll();
- return;
- }
-
- // How is the phone process doing?
- if (mPhoneReq != null) {
- if (mPhoneMemMonitor.checkLocked(curTime, mPhonePid,
- mPhonePss)) {
- // Just kill the phone process and let it restart.
- Slog.i(TAG, "Watchdog is killing the phone process");
- Process.killProcess(mPhonePid);
- }
- } else {
- mPhoneMemMonitor.clear();
- }
-
- needScheduledCheck = mNeedScheduledCheck;
- if (needScheduledCheck) {
- // Something is going bad, but now is not a good time to
- // tear things down... schedule an alarm to check again soon.
- nextTime = curTime + recheckInterval;
- if (nextTime < mMemcheckExecStartTime) {
- nextTime = mMemcheckExecStartTime;
- } else if (nextTime >= mMemcheckExecEndTime){
- // Need to check during next exec time... so that needs
- // to be computed.
- if (localLOGV) Slog.v(TAG, "Computing next time range");
- computeMemcheckTimesLocked(nextTime);
- nextTime = mMemcheckExecStartTime;
- }
-
- if (localLOGV) {
- mCalendar.setTimeInMillis(nextTime);
- Slog.v(TAG, "Next Alarm Time: " + mCalendar);
- }
- }
- }
-
- if (needScheduledCheck) {
- if (localLOGV) Slog.v(TAG, "Scheduling next memcheck alarm for "
- + ((nextTime-curTime)/1000/60) + "m from now");
- mAlarm.remove(mCheckupIntent);
- mAlarm.set(AlarmManager.RTC_WAKEUP, nextTime, mCheckupIntent);
- } else {
- if (localLOGV) Slog.v(TAG, "No need to schedule a memcheck alarm!");
- mAlarm.remove(mCheckupIntent);
- }
- }
-
- final PssStats mPssStats = new PssStats();
- final String[] mMemInfoFields = new String[] {
- "MemFree:", "Buffers:", "Cached:",
- "Active:", "Inactive:",
- "AnonPages:", "Mapped:", "Slab:",
- "SReclaimable:", "SUnreclaim:", "PageTables:" };
- final long[] mMemInfoSizes = new long[mMemInfoFields.length];
- final String[] mVMStatFields = new String[] {
- "pgfree ", "pgactivate ", "pgdeactivate ",
- "pgfault ", "pgmajfault " };
- final long[] mVMStatSizes = new long[mVMStatFields.length];
- final long[] mPrevVMStatSizes = new long[mVMStatFields.length];
- long mLastLogGlobalMemoryTime;
-
- void logGlobalMemory() {
- PssStats stats = mPssStats;
- mActivity.collectPss(stats);
- EventLog.writeEvent(EventLogTags.WATCHDOG_PSS_STATS,
- stats.mEmptyPss, stats.mEmptyCount,
- stats.mBackgroundPss, stats.mBackgroundCount,
- stats.mServicePss, stats.mServiceCount,
- stats.mVisiblePss, stats.mVisibleCount,
- stats.mForegroundPss, stats.mForegroundCount,
- stats.mNoPssCount);
- EventLog.writeEvent(EventLogTags.WATCHDOG_PROC_STATS,
- stats.mProcDeaths[0], stats.mProcDeaths[1], stats.mProcDeaths[2],
- stats.mProcDeaths[3], stats.mProcDeaths[4]);
- Process.readProcLines("/proc/meminfo", mMemInfoFields, mMemInfoSizes);
- for (int i=0; i<mMemInfoSizes.length; i++) {
- mMemInfoSizes[i] *= 1024;
- }
- EventLog.writeEvent(EventLogTags.WATCHDOG_MEMINFO,
- (int)mMemInfoSizes[0], (int)mMemInfoSizes[1], (int)mMemInfoSizes[2],
- (int)mMemInfoSizes[3], (int)mMemInfoSizes[4],
- (int)mMemInfoSizes[5], (int)mMemInfoSizes[6], (int)mMemInfoSizes[7],
- (int)mMemInfoSizes[8], (int)mMemInfoSizes[9], (int)mMemInfoSizes[10]);
- long now = SystemClock.uptimeMillis();
- long dur = now - mLastLogGlobalMemoryTime;
- mLastLogGlobalMemoryTime = now;
- Process.readProcLines("/proc/vmstat", mVMStatFields, mVMStatSizes);
- for (int i=0; i<mVMStatSizes.length; i++) {
- long v = mVMStatSizes[i];
- mVMStatSizes[i] -= mPrevVMStatSizes[i];
- mPrevVMStatSizes[i] = v;
- }
- EventLog.writeEvent(EventLogTags.WATCHDOG_VMSTAT, dur,
- (int)mVMStatSizes[0], (int)mVMStatSizes[1], (int)mVMStatSizes[2],
- (int)mVMStatSizes[3], (int)mVMStatSizes[4]);
- }
-
void checkReboot(boolean fromAlarm) {
int rebootInterval = mReqRebootInterval >= 0 ? mReqRebootInterval
: Settings.Secure.getInt(
@@ -730,47 +360,6 @@
return null;
}
- /**
- * Compute the times during which we next would like to perform process
- * restarts.
- *
- * @param curTime The current system time.
- */
- void computeMemcheckTimesLocked(long curTime) {
- if (mMemcheckLastTime == curTime) {
- return;
- }
-
- mMemcheckLastTime = curTime;
-
- long memcheckExecStartTime = Settings.Secure.getLong(
- mResolver, Settings.Secure.MEMCHECK_EXEC_START_TIME,
- MEMCHECK_DEFAULT_EXEC_START_TIME);
- long memcheckExecEndTime = Settings.Secure.getLong(
- mResolver, Settings.Secure.MEMCHECK_EXEC_END_TIME,
- MEMCHECK_DEFAULT_EXEC_END_TIME);
-
- mMemcheckExecEndTime = computeCalendarTime(mCalendar, curTime,
- memcheckExecEndTime);
- if (mMemcheckExecEndTime < curTime) {
- memcheckExecStartTime += 24*60*60;
- memcheckExecEndTime += 24*60*60;
- mMemcheckExecEndTime = computeCalendarTime(mCalendar, curTime,
- memcheckExecEndTime);
- }
- mMemcheckExecStartTime = computeCalendarTime(mCalendar, curTime,
- memcheckExecStartTime);
-
- if (localLOGV) {
- mCalendar.setTimeInMillis(curTime);
- Slog.v(TAG, "Current Time: " + mCalendar);
- mCalendar.setTimeInMillis(mMemcheckExecStartTime);
- Slog.v(TAG, "Start Check Time: " + mCalendar);
- mCalendar.setTimeInMillis(mMemcheckExecEndTime);
- Slog.v(TAG, "End Check Time: " + mCalendar);
- }
- }
-
static long computeCalendarTime(Calendar c, long curTime,
long secondsSinceMidnight) {