Handle losing conneciton with VHAL in Car Service

Test: TODO

Change-Id: Ib465b146db119bc4dbf6019f76619935891fd45d
Fix: b/34083734
diff --git a/service/src/com/android/car/CanBusErrorNotifier.java b/service/src/com/android/car/CanBusErrorNotifier.java
index 56be067..7caa1a8 100644
--- a/service/src/com/android/car/CanBusErrorNotifier.java
+++ b/service/src/com/android/car/CanBusErrorNotifier.java
@@ -23,6 +23,9 @@
 
 import com.android.internal.annotations.GuardedBy;
 
+import java.util.HashSet;
+import java.util.Set;
+
 /**
  * Class used to notify user about CAN bus failure.
  */
@@ -34,8 +37,10 @@
     private final Context mContext;
     private final NotificationManager mNotificationManager;
 
+    // Contains a set of objects that reported failure. The notification will be hidden only when
+    // this set is empty (all reported objects are in love and peace with the vehicle).
     @GuardedBy("this")
-    private boolean mFailed;
+    private final Set<Object> mReportedObjects = new HashSet<>();
 
     @GuardedBy("this")
     private Notification mNotification;
@@ -46,16 +51,29 @@
         mContext = context;
     }
 
-    public void setCanBusFailure(boolean failed) {
+    public void removeFailureReport(Object sender) {
+        setCanBusFailure(false, sender);
+    }
+
+    public void reportFailure(Object sender) {
+        setCanBusFailure(true, sender);
+    }
+
+    private void setCanBusFailure(boolean failed, Object sender) {
+        boolean shouldShowNotification;
         synchronized (this) {
-            if (mFailed == failed) {
+            boolean changed = failed
+                    ? mReportedObjects.add(sender) : mReportedObjects.remove(sender);
+
+            if (!changed) {
                 return;
             }
-            mFailed = failed;
-        }
-        Log.i(TAG, "Changing CAN bus failure state to " + failed);
 
-        if (failed) {
+            shouldShowNotification = !mReportedObjects.isEmpty();
+        }
+        Log.i(TAG, "Changing CAN bus failure state to " + shouldShowNotification);
+
+        if (shouldShowNotification) {
             showNotification();
         } else {
             hideNotification();
diff --git a/service/src/com/android/car/CarAudioService.java b/service/src/com/android/car/CarAudioService.java
index 26004be..424a611 100644
--- a/service/src/com/android/car/CarAudioService.java
+++ b/service/src/com/android/car/CarAudioService.java
@@ -167,7 +167,7 @@
                     CarAudioAttributesUtil.CAR_AUDIO_USAGE_CARSERVICE_CAR_PROXY);
 
     public CarAudioService(Context context, AudioHalService audioHal,
-            CarInputService inputService) {
+            CarInputService inputService, CanBusErrorNotifier errorNotifier) {
         mAudioHal = audioHal;
         mContext = context;
         mFocusHandlerThread = new HandlerThread(CarLog.TAG_AUDIO);
@@ -175,7 +175,7 @@
         mFocusHandlerThread.start();
         mFocusHandler = new CarAudioFocusChangeHandler(mFocusHandlerThread.getLooper());
         mAudioManager = (AudioManager) mContext.getSystemService(Context.AUDIO_SERVICE);
-        mCanBusErrorNotifier = new CanBusErrorNotifier(context);
+        mCanBusErrorNotifier =  errorNotifier;
         Resources res = context.getResources();
         mFocusResponseWaitTimeoutMs = (long) res.getInteger(R.integer.audioFocusWaitTimeoutMs);
         mNumConsecutiveHalFailuresForCanError =
@@ -1381,8 +1381,11 @@
 
     private void checkCanStatus() {
         // If CAN bus recovers, message will be removed.
-        mCanBusErrorNotifier.setCanBusFailure(
-                mNumConsecutiveHalFailures >= mNumConsecutiveHalFailuresForCanError);
+        if (mNumConsecutiveHalFailures >= mNumConsecutiveHalFailuresForCanError) {
+            mCanBusErrorNotifier.reportFailure(this);
+        } else {
+            mCanBusErrorNotifier.removeFailureReport(this);
+        }
     }
 
     private static boolean isAudioAttributesSame(AudioAttributes one, AudioAttributes two) {
diff --git a/service/src/com/android/car/CarService.java b/service/src/com/android/car/CarService.java
index b2f0edd..e7fa823 100644
--- a/service/src/com/android/car/CarService.java
+++ b/service/src/com/android/car/CarService.java
@@ -23,13 +23,20 @@
 import android.content.pm.PackageManager;
 import android.hardware.vehicle.V2_0.IVehicle;
 import android.os.Binder;
+import android.os.Build;
 import android.os.IBinder;
+import android.os.IHwBinder.DeathRecipient;
 import android.os.RemoteException;
+import android.os.SystemClock;
 import android.os.SystemProperties;
 import android.util.Log;
 
+import com.android.internal.annotations.VisibleForTesting;
+import com.android.internal.util.RingBufferIndices;
+
 import java.io.FileDescriptor;
 import java.io.PrintWriter;
+import java.util.NoSuchElementException;
 
 public class CarService extends Service {
 
@@ -38,19 +45,46 @@
 
     private static final long WAIT_FOR_VEHICLE_HAL_TIMEOUT_MS = 10_000;
 
+    private static final boolean IS_USER_BUILD = "user".equals(Build.TYPE);
+
+    private CanBusErrorNotifier mCanBusErrorNotifier;
     private ICarImpl mICarImpl;
+    private IVehicle mVehicle;
+
+    // If 10 crashes of Vehicle HAL occurred within 10 minutes then thrown an exception in
+    // Car Service.
+    private final CrashTracker mVhalCrashTracker = new CrashTracker(
+            10,  // Max crash count.
+            10 * 60 * 1000,  // 10 minutes - sliding time window.
+            () -> {
+                if (IS_USER_BUILD) {
+                    Log.e(CarLog.TAG_SERVICE, "Vehicle HAL keeps crashing, notifying user...");
+                    mCanBusErrorNotifier.reportFailure(CarService.this);
+                } else {
+                    throw new RuntimeException(
+                            "Vehicle HAL crashed too many times in a given time frame");
+                }
+            }
+    );
+
+    private final VehicleDeathRecipient mVehicleDeathRecipient = new VehicleDeathRecipient();
 
     @Override
     public void onCreate() {
         Log.i(CarLog.TAG_SERVICE, "Service onCreate");
-        IVehicle vehicle = getVehicle(WAIT_FOR_VEHICLE_HAL_TIMEOUT_MS);
-        if (vehicle == null) {
+        mCanBusErrorNotifier = new CanBusErrorNotifier(this /* context */);
+        mVehicle = getVehicleWithTimeout(WAIT_FOR_VEHICLE_HAL_TIMEOUT_MS);
+        if (mVehicle == null) {
             throw new IllegalStateException("Vehicle HAL service is not available.");
         }
 
-        mICarImpl = new ICarImpl(this, vehicle, SystemInterface.getDefault(this));
+        mICarImpl = new ICarImpl(this, mVehicle, SystemInterface.getDefault(this),
+                mCanBusErrorNotifier);
         mICarImpl.init();
         SystemProperties.set("boot.car_service_created", "1");
+
+        linkToDeath(mVehicle, mVehicleDeathRecipient);
+
         super.onCreate();
     }
 
@@ -58,6 +92,17 @@
     public void onDestroy() {
         Log.i(CarLog.TAG_SERVICE, "Service onDestroy");
         mICarImpl.release();
+        mCanBusErrorNotifier.removeFailureReport(this);
+
+        if (mVehicle != null) {
+            try {
+                mVehicle.unlinkToDeath(mVehicleDeathRecipient);
+                mVehicle = null;
+            } catch (RemoteException e) {
+                // Ignore errors on shutdown path.
+            }
+        }
+
         super.onDestroy();
     }
 
@@ -84,13 +129,17 @@
         if (args == null || args.length == 0) {
             writer.println("*dump car service*");
             mICarImpl.dump(writer);
+
+            writer.println("**Debug info**");
+            writer.println("Vehicle HAL reconnected: "
+                    + mVehicleDeathRecipient.deathCount + " times.");
         } else {
             mICarImpl.execShellCmd(args, writer);
         }
     }
 
     @Nullable
-    private IVehicle getVehicle(long waitMilliseconds) {
+    private IVehicle getVehicleWithTimeout(long waitMilliseconds) {
         IVehicle vehicle = getVehicle();
         long start = elapsedRealtime();
         while (vehicle == null && (start + waitMilliseconds) > elapsedRealtime()) {
@@ -102,6 +151,11 @@
 
             vehicle = getVehicle();
         }
+
+        if (vehicle != null) {
+            mCanBusErrorNotifier.removeFailureReport(this);
+        }
+
         return vehicle;
     }
 
@@ -110,8 +164,83 @@
         try {
             return IVehicle.getService(VEHICLE_SERVICE_NAME);
         } catch (RemoteException e) {
-            // TODO(pavelm)
-            return null;
+            Log.e(CarLog.TAG_SERVICE, "Failed to get IVehicle service", e);
+        } catch (NoSuchElementException e) {
+            Log.e(CarLog.TAG_SERVICE, "IVehicle service not registered yet");
+        }
+        return null;
+    }
+
+    private class VehicleDeathRecipient implements DeathRecipient {
+        private int deathCount = 0;
+
+        @Override
+        public void serviceDied(long cookie) {
+            Log.w(CarLog.TAG_SERVICE, "Vehicle HAL died.");
+
+            try {
+                mVehicle.unlinkToDeath(this);
+            } catch (RemoteException e) {
+                Log.e(CarLog.TAG_SERVICE, "Failed to unlinkToDeath", e);  // Log and continue.
+            }
+            mVehicle = null;
+
+            mVhalCrashTracker.crashDetected();
+
+            Log.i(CarLog.TAG_SERVICE, "Trying to reconnect to Vehicle HAL...");
+            mVehicle = getVehicleWithTimeout(WAIT_FOR_VEHICLE_HAL_TIMEOUT_MS);
+            if (mVehicle == null) {
+                throw new IllegalStateException("Failed to reconnect to Vehicle HAL");
+            }
+
+            linkToDeath(mVehicle, this);
+
+            Log.i(CarLog.TAG_SERVICE, "Notifying car service Vehicle HAL reconnected...");
+            mICarImpl.vehicleHalReconnected(mVehicle);
+        }
+    }
+
+    private static void linkToDeath(IVehicle vehicle, DeathRecipient recipient) {
+        try {
+            vehicle.linkToDeath(recipient, 0);
+        } catch (RemoteException e) {
+            throw new IllegalStateException("Failed to linkToDeath Vehicle HAL");
+        }
+    }
+
+    @VisibleForTesting
+    static class CrashTracker {
+        private final int mMaxCrashCountLimit;
+        private final int mSlidingWindowMillis;
+
+        private final long[] mCrashTimestamps;
+        private final RingBufferIndices mCrashTimestampsIndices;
+        private final Runnable mCallback;
+
+        /**
+         * If maxCrashCountLimit number of crashes occurred within slidingWindowMillis time
+         * frame then call provided callback function.
+         */
+        CrashTracker(int maxCrashCountLimit, int slidingWindowMillis, Runnable callback) {
+            mMaxCrashCountLimit = maxCrashCountLimit;
+            mSlidingWindowMillis = slidingWindowMillis;
+            mCallback = callback;
+
+            mCrashTimestamps = new long[maxCrashCountLimit];
+            mCrashTimestampsIndices = new RingBufferIndices(mMaxCrashCountLimit);
+        }
+
+        void crashDetected() {
+            long lastCrash = SystemClock.elapsedRealtime();
+            mCrashTimestamps[mCrashTimestampsIndices.add()] = lastCrash;
+
+            if (mCrashTimestampsIndices.size() == mMaxCrashCountLimit) {
+                long firstCrash = mCrashTimestamps[mCrashTimestampsIndices.indexOf(0)];
+
+                if (lastCrash - firstCrash < mSlidingWindowMillis) {
+                    mCallback.run();
+                }
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/service/src/com/android/car/CarServiceBase.java b/service/src/com/android/car/CarServiceBase.java
index 5745b10..c4b578d 100644
--- a/service/src/com/android/car/CarServiceBase.java
+++ b/service/src/com/android/car/CarServiceBase.java
@@ -32,5 +32,8 @@
     /** service should stop and all resources should be released. */
     void release();
 
+    /** Called when connection to Vehicle HAL was restored. */
+    default void vehicleHalReconnected() {}
+
     void dump(PrintWriter writer);
 }
diff --git a/service/src/com/android/car/ICarImpl.java b/service/src/com/android/car/ICarImpl.java
index e05efaa..ac0263e 100644
--- a/service/src/com/android/car/ICarImpl.java
+++ b/service/src/com/android/car/ICarImpl.java
@@ -50,7 +50,7 @@
     }
 
     private final Context mContext;
-    private VehicleHal mHal;
+    private final VehicleHal mHal;
 
     private final SystemActivityMonitoringService mSystemActivityMonitoringService;
     private final CarPowerManagementService mCarPowerManagementService;
@@ -77,7 +77,8 @@
     @GuardedBy("this")
     private CarTestService mCarTestService;
 
-    public ICarImpl(Context serviceContext, IVehicle vehicle, SystemInterface systemInterface) {
+    public ICarImpl(Context serviceContext, IVehicle vehicle, SystemInterface systemInterface,
+            CanBusErrorNotifier errorNotifier) {
         mContext = serviceContext;
         mHal = new VehicleHal(vehicle);
         mSystemActivityMonitoringService = new SystemActivityMonitoringService(serviceContext);
@@ -92,7 +93,7 @@
         mCarInfoService = new CarInfoService(serviceContext, mHal.getInfoHal());
         mAppFocusService = new AppFocusService(serviceContext, mSystemActivityMonitoringService);
         mCarAudioService = new CarAudioService(serviceContext, mHal.getAudioHal(),
-                mCarInputService);
+                mCarInputService, errorNotifier);
         mCarCabinService = new CarCabinService(serviceContext, mHal.getCabinHal());
         mCarHvacService = new CarHvacService(serviceContext, mHal.getHvacHal());
         mCarRadioService = new CarRadioService(serviceContext, mHal.getRadioHal());
@@ -143,6 +144,13 @@
         mHal.release();
     }
 
+    public void vehicleHalReconnected(IVehicle vehicle) {
+        mHal.vehicleHalReconnected(vehicle);
+        for (CarServiceBase service : mAllServices) {
+            service.vehicleHalReconnected();
+        }
+    }
+
     @Override
     public IBinder getCarService(String serviceName) {
         switch (serviceName) {
diff --git a/service/src/com/android/car/hal/HalClient.java b/service/src/com/android/car/hal/HalClient.java
index 4da68f1..01d576f 100644
--- a/service/src/com/android/car/hal/HalClient.java
+++ b/service/src/com/android/car/hal/HalClient.java
@@ -19,7 +19,6 @@
 import static android.os.SystemClock.elapsedRealtime;
 
 import android.hardware.vehicle.V2_0.IVehicle;
-import android.hardware.vehicle.V2_0.IVehicle.getCallback;
 import android.hardware.vehicle.V2_0.IVehicleCallback;
 import android.hardware.vehicle.V2_0.StatusCode;
 import android.hardware.vehicle.V2_0.SubscribeFlags;
@@ -84,9 +83,15 @@
         mVehicle.unsubscribe(mInternalCallback, prop);
     }
 
-    public void setValue(VehiclePropValue propValue) throws RemoteException, PropertyTimeoutException {
-        int status = invokeRetriable(() -> mVehicle.set(propValue),
-                WAIT_CAP_FOR_RETRIABLE_RESULT_MS, SLEEP_BETWEEN_RETRIABLE_INVOKES_MS);
+    public void setValue(VehiclePropValue propValue) throws PropertyTimeoutException {
+        int status = invokeRetriable(() -> {
+            try {
+                return mVehicle.set(propValue);
+            } catch (RemoteException e) {
+                Log.e(CarLog.TAG_HAL, "Failed to set value", e);
+                return StatusCode.TRY_AGAIN;
+            }
+        }, WAIT_CAP_FOR_RETRIABLE_RESULT_MS, SLEEP_BETWEEN_RETRIABLE_INVOKES_MS);
 
         if (StatusCode.INVALID_ARG == status) {
             throw new IllegalArgumentException(
@@ -105,7 +110,7 @@
         }
     }
 
-    VehiclePropValue getValue(VehiclePropValue requestedPropValue) throws RemoteException, PropertyTimeoutException {
+    VehiclePropValue getValue(VehiclePropValue requestedPropValue) throws PropertyTimeoutException {
         final ObjectWrapper<VehiclePropValue> valueWrapper = new ObjectWrapper<>();
         int status = invokeRetriable(() -> {
             ValueResult res = internalGet(requestedPropValue);
@@ -133,26 +138,28 @@
         return valueWrapper.object;
     }
 
-    private ValueResult internalGet(VehiclePropValue requestedPropValue) throws RemoteException {
+    private ValueResult internalGet(VehiclePropValue requestedPropValue) {
         final ValueResult result = new ValueResult();
-        mVehicle.get(requestedPropValue,
-                new getCallback() {
-                    @Override
-                    public void onValues(int status, VehiclePropValue propValue) {
+        try {
+            mVehicle.get(requestedPropValue,
+                    (status, propValue) -> {
                         result.status = status;
                         result.propValue = propValue;
-                    }
-                });
+                    });
+        } catch (RemoteException e) {
+            Log.e(CarLog.TAG_HAL, "Failed to get value from vehicle HAL", e);
+            result.status = StatusCode.TRY_AGAIN;
+        }
 
         return result;
     }
 
     interface RetriableCallback {
         /** Returns {@link StatusCode} */
-        int action() throws RemoteException;
+        int action();
     }
 
-    private static int invokeRetriable(RetriableCallback callback, long timeoutMs, long sleepMs) throws RemoteException {
+    private static int invokeRetriable(RetriableCallback callback, long timeoutMs, long sleepMs) {
         int status = callback.action();
         long startTime = elapsedRealtime();
         while (StatusCode.TRY_AGAIN == status && (elapsedRealtime() - startTime) < timeoutMs) {
diff --git a/service/src/com/android/car/hal/VehicleHal.java b/service/src/com/android/car/hal/VehicleHal.java
index 2d40209..4f78812 100644
--- a/service/src/com/android/car/hal/VehicleHal.java
+++ b/service/src/com/android/car/hal/VehicleHal.java
@@ -48,6 +48,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map.Entry;
 import java.util.Set;
 
 /**
@@ -72,13 +73,15 @@
     private final HvacHalService mHvacHal;
     private final InputHalService mInputHal;
     private final VendorExtensionHalService mVendorExtensionHal;
-    private final HalClient mHalClient;
 
-    /** stores handler for each HAL property. Property events are sent to handler. */
+    /** Might be re-assigned if Vehicle HAL is reconnected. */
+    private volatile HalClient mHalClient;
+
+    /** Stores handler for each HAL property. Property events are sent to handler. */
     private final SparseArray<HalServiceBase> mPropertyHandlers = new SparseArray<>();
     /** This is for iterating all HalServices with fixed order. */
     private final HalServiceBase[] mAllServices;
-    private final ArraySet<Integer> mSubscribedProperties = new ArraySet<>();
+    private final HashMap<Integer, Float> mSubscribedProperties = new HashMap<>();
     private final HashMap<Integer, VehiclePropConfig> mAllProperties = new HashMap<>();
     private final HashMap<Integer, VehiclePropertyEventInfo> mEventLog = new HashMap<>();
 
@@ -129,13 +132,29 @@
         mHalClient = halClient;
     }
 
+    public void vehicleHalReconnected(IVehicle vehicle) {
+        synchronized (this) {
+            mHalClient = new HalClient(vehicle, mHandlerThread.getLooper(),
+                    this /*IVehicleCallback*/);
+
+            for (Entry<Integer, Float> subscription : mSubscribedProperties.entrySet()) {
+                try {
+                    mHalClient.subscribe(subscription.getKey(), subscription.getValue());
+                } catch (RemoteException e) {
+                    throw new RuntimeException("Unable to subscribe to property: 0x"
+                            + toHexString(subscription.getKey()) + " with "
+                            + subscription.getValue() + "Hz", e);
+                }
+            }
+        }
+    }
+
     public void init() {
         Set<VehiclePropConfig> properties;
         try {
             properties = new HashSet<>(mHalClient.getAllPropConfigs());
         } catch (RemoteException e) {
-            // TODO(pavelm)
-            properties = new HashSet<>();
+            throw new RuntimeException("Unable to retrieve vehicle property configuration", e);
         }
 
         synchronized (this) {
@@ -169,11 +188,12 @@
             mAllServices[i].release();
         }
         synchronized (this) {
-            for (int p : mSubscribedProperties) {
+            for (int p : mSubscribedProperties.keySet()) {
                 try {
                     mHalClient.unsubscribe(p);
                 } catch (RemoteException e) {
-                    // TODO(pavelm)
+                    //  Ignore exceptions on shutdown path.
+                    Log.w(CarLog.TAG_HAL, "Failed to unsubscribe", e);
                 }
             }
             mSubscribedProperties.clear();
@@ -245,12 +265,12 @@
         } else if (isPropertySubscribable(config)) {
             synchronized (this) {
                 assertServiceOwnerLocked(service, property);
-                mSubscribedProperties.add(property);
+                mSubscribedProperties.put(property, samplingRateHz);
             }
             try {
                 mHalClient.subscribe(property, samplingRateHz);
             } catch (RemoteException e) {
-                // TODO(pavelm)
+                Log.e(CarLog.TAG_HAL, "Failed to subscribe to property: 0x" + property, e);
             }
         } else {
             Log.e(CarLog.TAG_HAL, "Cannot subscribe to property: " + property);
@@ -277,7 +297,8 @@
             try {
                 mHalClient.unsubscribe(property);
             } catch (RemoteException e) {
-                // TODO(pavelm)
+                Log.e(CarLog.TAG_SERVICE, "Failed to unsubscribe from property: 0x"
+                        + toHexString(property), e);
             }
         } else {
             Log.e(CarLog.TAG_HAL, "Cannot unsubscribe property: " + property);
@@ -302,12 +323,7 @@
         VehiclePropValue propValue = new VehiclePropValue();
         propValue.prop = propertyId;
         propValue.areaId = areaId;
-        try {
-            return mHalClient.getValue(propValue);
-        } catch (RemoteException e) {
-            // TODO(pavelm)
-            return null;
-        }
+        return mHalClient.getValue(propValue);
     }
 
     public <T> T get(Class clazz, int propertyId) throws PropertyTimeoutException {
@@ -322,12 +338,8 @@
     public <T> T get(Class clazz, VehiclePropValue requestedPropValue)
             throws PropertyTimeoutException {
         VehiclePropValue propValue;
-        try {
-            propValue = mHalClient.getValue(requestedPropValue);
-        } catch (RemoteException e) {
-            // TODO(pavelm)
-            return null;
-        }
+        propValue = mHalClient.getValue(requestedPropValue);
+
         if (clazz == Integer.class || clazz == int.class) {
             return (T) propValue.value.int32Values.get(0);
         } else if (clazz == Boolean.class || clazz == boolean.class) {
@@ -355,20 +367,11 @@
 
     public VehiclePropValue get(VehiclePropValue requestedPropValue)
             throws PropertyTimeoutException {
-        try {
-            return mHalClient.getValue(requestedPropValue);
-        } catch (RemoteException e) {
-            // TODO(pavelm)
-            return null;
-        }
+        return mHalClient.getValue(requestedPropValue);
     }
 
     void set(VehiclePropValue propValue) throws PropertyTimeoutException {
-        try {
-            mHalClient.setValue(propValue);
-        } catch (RemoteException e) {
-            // TODO(pavelm)
-        }
+        mHalClient.setValue(propValue);
     }
 
     @CheckResult
@@ -534,13 +537,11 @@
         void submit() throws PropertyTimeoutException {
             HalClient client =  mClient.get();
             if (client != null) {
-                Log.i(CarLog.TAG_HAL, "set, property: 0x" + toHexString(mPropValue.prop)
-                        + ", areaId: 0x" + toHexString(mPropValue.areaId));
-                try {
-                    client.setValue(mPropValue);
-                } catch (RemoteException e) {
-                    // TODO(pavelm)
+                if (DBG) {
+                    Log.i(CarLog.TAG_HAL, "set, property: 0x" + toHexString(mPropValue.prop)
+                            + ", areaId: 0x" + toHexString(mPropValue.areaId));
                 }
+                client.setValue(mPropValue);
             }
         }
     }
diff --git a/tests/EmbeddedKitchenSinkApp/src/com/google/android/car/kitchensink/CarEmulator.java b/tests/EmbeddedKitchenSinkApp/src/com/google/android/car/kitchensink/CarEmulator.java
index 9875049..fa8ac61 100644
--- a/tests/EmbeddedKitchenSinkApp/src/com/google/android/car/kitchensink/CarEmulator.java
+++ b/tests/EmbeddedKitchenSinkApp/src/com/google/android/car/kitchensink/CarEmulator.java
@@ -18,6 +18,8 @@
 
 import static android.hardware.vehicle.V2_0.VehicleProperty.AUDIO_VOLUME_LIMIT;
 
+import com.google.android.collect.Lists;
+
 import android.car.Car;
 import android.content.Context;
 import android.hardware.vehicle.V2_0.VehicleAudioFocusIndex;
@@ -32,8 +34,6 @@
 import android.os.SystemClock;
 import android.util.SparseIntArray;
 
-import com.google.android.collect.Lists;
-
 import com.android.car.ICarImpl;
 import com.android.car.SystemInterface;
 import com.android.car.vehiclehal.VehiclePropValueBuilder;
@@ -66,7 +66,7 @@
     private CarEmulator(Context context) {
         mHalEmulator = new MockedVehicleHal();
         ICarImpl carService = new ICarImpl(context, mHalEmulator,
-                SystemInterface.getDefault(context));
+                SystemInterface.getDefault(context), null /* error notifier */);
         mCar = new Car(context, carService, null /* Handler */);
     }
 
diff --git a/tests/carservice_test/src/com/android/car/test/MockedCarTestBase.java b/tests/carservice_test/src/com/android/car/test/MockedCarTestBase.java
index 0c3f84b..d1b08ac 100644
--- a/tests/carservice_test/src/com/android/car/test/MockedCarTestBase.java
+++ b/tests/carservice_test/src/com/android/car/test/MockedCarTestBase.java
@@ -89,7 +89,8 @@
 
         mFakeSystemInterface = new FakeSystemInterface();
         Context context = getCarServiceContext();
-        mCarImpl = new ICarImpl(context, mMockedVehicleHal, mFakeSystemInterface);
+        mCarImpl = new ICarImpl(context, mMockedVehicleHal, mFakeSystemInterface,
+                null /* error notifier */);
 
         initMockedHal(false /* no need to release */);
 
diff --git a/tests/carservice_unit_test/src/com/android/car/test/CarPowerManagementServiceTest.java b/tests/carservice_unit_test/src/com/android/car/CarPowerManagementServiceTest.java
similarity index 99%
rename from tests/carservice_unit_test/src/com/android/car/test/CarPowerManagementServiceTest.java
rename to tests/carservice_unit_test/src/com/android/car/CarPowerManagementServiceTest.java
index bb462ee..c092566 100644
--- a/tests/carservice_unit_test/src/com/android/car/test/CarPowerManagementServiceTest.java
+++ b/tests/carservice_unit_test/src/com/android/car/CarPowerManagementServiceTest.java
@@ -14,17 +14,15 @@
  * limitations under the License.
  */
 
-package com.android.car.test;
+package com.android.car;
 
 import android.test.AndroidTestCase;
 import android.util.Log;
 
-import com.android.car.CarPowerManagementService;
 import com.android.car.CarPowerManagementService.PowerEventProcessingHandler;
 import com.android.car.CarPowerManagementService.PowerServiceEventListener;
 import com.android.car.hal.PowerHalService;
 import com.android.car.hal.PowerHalService.PowerState;
-import com.android.car.SystemInterface;
 
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
diff --git a/tests/carservice_unit_test/src/com/android/car/CrashTrackerTest.java b/tests/carservice_unit_test/src/com/android/car/CrashTrackerTest.java
new file mode 100644
index 0000000..1ac320f
--- /dev/null
+++ b/tests/carservice_unit_test/src/com/android/car/CrashTrackerTest.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.car;
+
+
+import android.test.AndroidTestCase;
+import android.util.MutableBoolean;
+
+import com.android.car.CarService.CrashTracker;
+
+public class CrashTrackerTest extends AndroidTestCase {
+
+    public void testCrashingTooManyTimes() throws InterruptedException {
+        final int SLIDING_WINDOW = 100;
+        final MutableBoolean callbackTriggered = new MutableBoolean(false);
+
+        CarService.CrashTracker crashTracker = new CrashTracker(3, SLIDING_WINDOW,
+                () -> callbackTriggered.value = true);
+
+
+        crashTracker.crashDetected();
+        crashTracker.crashDetected();
+        Thread.sleep(SLIDING_WINDOW + 1);
+        crashTracker.crashDetected();
+
+        assertFalse(callbackTriggered.value);
+
+        crashTracker.crashDetected();
+        assertFalse(callbackTriggered.value);
+        crashTracker.crashDetected();
+        assertTrue(callbackTriggered.value);  // Last 3 crashes should be within SLIDING_WINDOW
+                                              // time frame.
+
+    }
+}
diff --git a/tests/carservice_unit_test/src/com/android/car/test/MockedPowerHalService.java b/tests/carservice_unit_test/src/com/android/car/MockedPowerHalService.java
similarity index 96%
rename from tests/carservice_unit_test/src/com/android/car/test/MockedPowerHalService.java
rename to tests/carservice_unit_test/src/com/android/car/MockedPowerHalService.java
index 814048e..02c23f2 100644
--- a/tests/carservice_unit_test/src/com/android/car/test/MockedPowerHalService.java
+++ b/tests/carservice_unit_test/src/com/android/car/MockedPowerHalService.java
@@ -13,13 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package com.android.car.test;
+package com.android.car;
 
 import android.util.Log;
 
 import com.android.car.hal.PowerHalService;
-import com.android.car.hal.PowerHalService.PowerEventListener;
-import com.android.car.hal.PowerHalService.PowerState;
 import com.android.car.hal.VehicleHal;
 
 import java.util.LinkedList;