shill: cellular: Use ConnectionHealthChecker for out-of-credit detection

Added code to trigger active connection monitoring using
ConnectionHealthChecker to detect out-of-credit scenarios, when:

  1. TrafficMonitor detects congestion and thus a possible out-of-credit
     situation;

  2. Portal detection fails after Cellular connects.

BUG=chromium:225912,225915
TEST=1. Build and run unit tests.
     2. While using a link device, run it out of credit. An
        out-of-credit dialog should pop up and the service should
        disconnect,
     3. While on a link device with an out-of-credit SIM, try to connect
        to Cellular. If it successfully connects and the connection
        does not drop, an out-of-credit dialog should pop up and the
        service should disconnect.

Change-Id: If70069c3630757b74099a4cefdfe6c67d8d308de
Reviewed-on: https://gerrit.chromium.org/gerrit/47717
Reviewed-by: Thieu Le <thieule@chromium.org>
Commit-Queue: Arman Uguray <armansito@chromium.org>
Tested-by: Arman Uguray <armansito@chromium.org>
diff --git a/Makefile b/Makefile
index 3643a5d..aff65c5 100644
--- a/Makefile
+++ b/Makefile
@@ -366,6 +366,7 @@
 	mock_async_connection.o \
 	mock_certificate_file.o \
 	mock_connection.o \
+	mock_connection_health_checker.o \
 	mock_control.o \
 	mock_dbus_manager.o \
 	mock_dbus_objectmanager_proxy.o \
diff --git a/cellular.cc b/cellular.cc
index ddb834b..c7510ec 100644
--- a/cellular.cc
+++ b/cellular.cc
@@ -257,7 +257,7 @@
     // Registration state updates may have been ignored while the
     // modem was not yet marked enabled.
     HandleNewRegistrationState();
-    if (capability_->ShouldEnableTrafficMonitoring())
+    if (capability_->ShouldDetectOutOfCredit())
       set_traffic_monitor_enabled(true);
   }
   callback.Run(error);
@@ -356,7 +356,36 @@
 void Cellular::OnNoNetworkRouting() {
   SLOG(Cellular, 2) << __func__;
   Device::OnNoNetworkRouting();
-  // TODO(armansito): Initiate active probing.
+
+  SLOG(Cellular, 2) << "Requesting active probe for out-of-credit detection.";
+  RequestConnectionHealthCheck();
+}
+
+void Cellular::OnConnectionHealthCheckerResult(
+      ConnectionHealthChecker::Result result) {
+  SLOG(Cellular, 2) << __func__ << "(Result = " << result << ")";
+
+  if (result == ConnectionHealthChecker::kResultElongatedTimeWait ||
+      result == ConnectionHealthChecker::kResultCongestedTxQueue) {
+    SLOG(Cellular, 2) << "Active probe determined possible out-of-credits "
+                      << "scenario.";
+    if (service().get()) {
+      service()->SetOutOfCredits(true);
+      SLOG(Cellular, 2) << "Disconnecting due to out-of-credit scenario.";
+      Error error;
+      service()->Disconnect(&error);
+    }
+  }
+}
+
+void Cellular::PortalDetectorCallback(const PortalDetector::Result &result) {
+  Device::PortalDetectorCallback(result);
+  if (result.status != PortalDetector::kStatusSuccess &&
+      capability_->ShouldDetectOutOfCredit()) {
+    SLOG(Cellular, 2) << "Portal detection failed. Launching active probe for "
+                      << "out-of-credit detection.";
+    RequestConnectionHealthCheck();
+  }
 }
 
 void Cellular::Scan(ScanType scan_type, Error *error) {
diff --git a/cellular.h b/cellular.h
index f4c3410..72ed743 100644
--- a/cellular.h
+++ b/cellular.h
@@ -212,6 +212,10 @@
   // Is the underlying device in the process of activating?
   bool IsActivating() const;
 
+  virtual void OnConnectionHealthCheckerResult(
+      ConnectionHealthChecker::Result result);
+  virtual void PortalDetectorCallback(const PortalDetector::Result &result);
+
  private:
   friend class CellularTest;
   friend class CellularCapabilityTest;
@@ -264,6 +268,7 @@
   FRIEND_TEST(CellularTest, ModemStateChangeEnable);
   FRIEND_TEST(CellularTest, ModemStateChangeStaleConnected);
   FRIEND_TEST(CellularTest, ModemStateChangeValidConnected);
+  FRIEND_TEST(CellularTest, OnConnectionHealthCheckerResult);
   FRIEND_TEST(CellularTest, SetAllowRoaming);
   FRIEND_TEST(CellularTest, StartModemCallback);
   FRIEND_TEST(CellularTest, StartModemCallbackFail);
diff --git a/cellular_capability.cc b/cellular_capability.cc
index fad9537..641ddd5 100644
--- a/cellular_capability.cc
+++ b/cellular_capability.cc
@@ -108,7 +108,7 @@
   return false;
 }
 
-bool CellularCapability::ShouldEnableTrafficMonitoring() const {
+bool CellularCapability::ShouldDetectOutOfCredit() const {
   return false;
 }
 
diff --git a/cellular_capability.h b/cellular_capability.h
index 24846c6..3597df6 100644
--- a/cellular_capability.h
+++ b/cellular_capability.h
@@ -201,7 +201,7 @@
   // Returns true if the cellular device should initiate passive traffic
   // monitoring to trigger active out-of-credit detection checks. This
   // implementation returns false by default.
-  virtual bool ShouldEnableTrafficMonitoring() const;
+  virtual bool ShouldDetectOutOfCredit() const;
 
  protected:
   // Releases all proxies held by the object.  This is most useful
diff --git a/cellular_capability_universal.cc b/cellular_capability_universal.cc
index c9b4914..1c6d4cf 100644
--- a/cellular_capability_universal.cc
+++ b/cellular_capability_universal.cc
@@ -726,7 +726,7 @@
   return provider_requires_roaming_ || allow_roaming_property();
 }
 
-bool CellularCapabilityUniversal::ShouldEnableTrafficMonitoring() const {
+bool CellularCapabilityUniversal::ShouldDetectOutOfCredit() const {
   return model_id_ == kE362ModelId;
 }
 
diff --git a/cellular_capability_universal.h b/cellular_capability_universal.h
index d32994c..bdefb74 100644
--- a/cellular_capability_universal.h
+++ b/cellular_capability_universal.h
@@ -111,7 +111,7 @@
       const DBusPropertiesMap &changed_properties,
       const std::vector<std::string> &invalidated_properties);
   virtual bool AllowRoaming();
-  virtual bool ShouldEnableTrafficMonitoring() const;
+  virtual bool ShouldDetectOutOfCredit() const;
 
  protected:
   virtual void InitProxies();
@@ -167,7 +167,7 @@
   FRIEND_TEST(CellularCapabilityUniversalMainTest, ScanFailure);
   FRIEND_TEST(CellularCapabilityUniversalMainTest, SetHomeProvider);
   FRIEND_TEST(CellularCapabilityUniversalMainTest,
-              ShouldEnableTrafficMonitoring);
+              ShouldDetectOutOfCredit);
   FRIEND_TEST(CellularCapabilityUniversalMainTest, SimLockStatusChanged);
   FRIEND_TEST(CellularCapabilityUniversalMainTest, SimPathChanged);
   FRIEND_TEST(CellularCapabilityUniversalMainTest, SimPropertiesChanged);
diff --git a/cellular_capability_universal_unittest.cc b/cellular_capability_universal_unittest.cc
index 58c3945..0997827 100644
--- a/cellular_capability_universal_unittest.cc
+++ b/cellular_capability_universal_unittest.cc
@@ -1760,11 +1760,11 @@
             capability_->GetNetworkTechnologyString());
 }
 
-TEST_F(CellularCapabilityUniversalMainTest, ShouldEnableTrafficMonitoring) {
+TEST_F(CellularCapabilityUniversalMainTest, ShouldDetectOutOfCredit) {
   capability_->model_id_.clear();
-  EXPECT_FALSE(capability_->ShouldEnableTrafficMonitoring());
+  EXPECT_FALSE(capability_->ShouldDetectOutOfCredit());
   capability_->model_id_ = CellularCapabilityUniversal::kE362ModelId;
-  EXPECT_TRUE(capability_->ShouldEnableTrafficMonitoring());
+  EXPECT_TRUE(capability_->ShouldDetectOutOfCredit());
 }
 
 }  // namespace shill
diff --git a/cellular_service.h b/cellular_service.h
index 2feda3d..9608025 100644
--- a/cellular_service.h
+++ b/cellular_service.h
@@ -133,6 +133,7 @@
   FRIEND_TEST(CellularCapabilityUniversalMainTest, UpdateServiceName);
   FRIEND_TEST(CellularCapabilityUniversalMainTest, UpdateStorageIdentifier);
   FRIEND_TEST(CellularTest, Connect);
+  FRIEND_TEST(CellularTest, OnConnectionHealthCheckerResult);
   FRIEND_TEST(CellularServiceTest, SetApn);
   FRIEND_TEST(CellularServiceTest, ClearApn);
   FRIEND_TEST(CellularServiceTest, LastGoodApn);
diff --git a/cellular_unittest.cc b/cellular_unittest.cc
index ecff7be..df176f5 100644
--- a/cellular_unittest.cc
+++ b/cellular_unittest.cc
@@ -44,6 +44,7 @@
 using testing::_;
 using testing::AnyNumber;
 using testing::Invoke;
+using testing::Mock;
 using testing::NiceMock;
 using testing::Return;
 using testing::SetArgumentPointee;
@@ -384,7 +385,7 @@
 
   // Different tests simulate a cellular service being set using a real /m mock
   // service.
-  CellularService* SetService() {
+  CellularService *SetService() {
     device_->service_ = new CellularService(
         modem_info_.control_interface(),
         modem_info_.dispatcher(),
@@ -393,14 +394,15 @@
         device_);
     return device_->service_;
   }
-  CellularService* SetMockService() {
-    device_->service_ = new MockCellularService(
+  MockCellularService *SetMockService() {
+    MockCellularService *service = new MockCellularService(
         modem_info_.control_interface(),
         modem_info_.dispatcher(),
         modem_info_.metrics(),
         modem_info_.manager(),
         device_);
-    return device_->service_;
+    device_->service_ = service;
+    return service;
   }
 
   EventDispatcher dispatcher_;
@@ -831,7 +833,7 @@
                                    Unretained(this)),
                               Error(Error::kSuccess));
   EXPECT_FALSE(device_->traffic_monitor_enabled());
-  testing::Mock::VerifyAndClearExpectations(this);
+  Mock::VerifyAndClearExpectations(this);
 
   device_->state_ = Cellular::kStateDisabled;
 
@@ -841,7 +843,7 @@
                                    Unretained(this)),
                               Error(Error::kOperationFailed));
   EXPECT_FALSE(device_->traffic_monitor_enabled());
-  testing::Mock::VerifyAndClearExpectations(this);
+  Mock::VerifyAndClearExpectations(this);
 
   EXPECT_CALL(*this, TestCallback(IsSuccess()));
   device_->StartModemCallback(Bind(&CellularTest::TestCallback,
@@ -888,6 +890,38 @@
   EXPECT_FALSE(device_->service_.get());
 }
 
+TEST_F(CellularTest, OnConnectionHealthCheckerResult) {
+  scoped_refptr<MockCellularService> service(SetMockService());
+  EXPECT_FALSE(service->out_of_credits_);
+
+  EXPECT_CALL(*service, Disconnect(_)).Times(0);
+  device_->OnConnectionHealthCheckerResult(
+      ConnectionHealthChecker::kResultUnknown);
+  EXPECT_FALSE(service->out_of_credits_);
+  device_->OnConnectionHealthCheckerResult(
+      ConnectionHealthChecker::kResultInProgress);
+  EXPECT_FALSE(service->out_of_credits_);
+  device_->OnConnectionHealthCheckerResult(
+      ConnectionHealthChecker::kResultConnectionFailure);
+  EXPECT_FALSE(service->out_of_credits_);
+  Mock::VerifyAndClearExpectations(service);
+
+  EXPECT_CALL(*service, Disconnect(_)).Times(1);
+  device_->OnConnectionHealthCheckerResult(
+      ConnectionHealthChecker::kResultElongatedTimeWait);
+  EXPECT_TRUE(service->out_of_credits_);
+  Mock::VerifyAndClearExpectations(service);
+
+  service->out_of_credits_ = false;
+  EXPECT_CALL(*service, Disconnect(_)).Times(1);
+  device_->OnConnectionHealthCheckerResult(
+      ConnectionHealthChecker::kResultCongestedTxQueue);
+  EXPECT_TRUE(service->out_of_credits_);
+
+  // Don't leak the service object.
+  device_->service_ = NULL;
+}
+
 // The following test crashes under Clang
 // Disabling it for now (crosbug.com/39351)
 #if defined(__clang__)
diff --git a/connection_health_checker.cc b/connection_health_checker.cc
index a362942..54b077b 100644
--- a/connection_health_checker.cc
+++ b/connection_health_checker.cc
@@ -121,6 +121,8 @@
 
 void ConnectionHealthChecker::SetupTcpConnection() {
   IPAddress ip = remote_ips_.front();
+  SLOG(Connection, 3) << __func__ << ": Starting connection at "
+                      << ip.ToString();
   if (tcp_connection_->Start(ip, kRemotePort)) {
     // TCP connection successful, no need to try more.
     return;
diff --git a/connection_health_checker.h b/connection_health_checker.h
index 8aac9dc..f788878 100644
--- a/connection_health_checker.h
+++ b/connection_health_checker.h
@@ -63,11 +63,11 @@
 
   // A new ConnectionHealthChecker is created with a default URL to attempt the
   // TCP connection with. Add a URL to try.
-  void AddRemoteURL(const std::string &url_string);
+  virtual void AddRemoteURL(const std::string &url_string);
 
   // Name resolution can fail in conditions -(1)- and -(2)-. Add an IP address
   // to attempt the TCP connection with.
-  void AddRemoteIP(IPAddress ip);
+  virtual void AddRemoteIP(IPAddress ip);
 
   // Start a connection health check. The health check involves one or more
   // attempts at establishing and using a TCP connection. |result_callback_| is
@@ -87,6 +87,9 @@
   // Accessors.
   const IPAddressQueue &remote_ips() { return remote_ips_; }
   void set_run_data_test(bool val) { run_data_test_ = val; }
+  virtual bool health_check_in_progress() const {
+      return health_check_in_progress_;
+  }
 
  private:
   friend class ConnectionHealthCheckerTest;
diff --git a/device.cc b/device.cc
index 3c801c3..9151093 100644
--- a/device.cc
+++ b/device.cc
@@ -18,15 +18,19 @@
 #include <base/stringprintf.h>
 #include <chromeos/dbus/service_constants.h>
 
+#include "shill/async_connection.h"
 #include "shill/connection.h"
+#include "shill/connection_health_checker.h"
 #include "shill/control_interface.h"
 #include "shill/device_dbus_adaptor.h"
 #include "shill/dhcp_config.h"
 #include "shill/dhcp_provider.h"
+#include "shill/dns_client.h"
 #include "shill/error.h"
 #include "shill/event_dispatcher.h"
 #include "shill/geolocation_info.h"
 #include "shill/http_proxy.h"
+#include "shill/ip_address.h"
 #include "shill/link_monitor.h"
 #include "shill/logging.h"
 #include "shill/manager.h"
@@ -35,6 +39,7 @@
 #include "shill/refptr_types.h"
 #include "shill/rtnl_handler.h"
 #include "shill/service.h"
+#include "shill/socket_info_reader.h"
 #include "shill/store_interface.h"
 #include "shill/technology.h"
 #include "shill/traffic_monitor.h"
@@ -161,6 +166,13 @@
                                    &Device::GetTransmitByteCountProperty);
   }
 
+  // TODO(armansito): Remove this hack once shill has support for caching DNS
+  // results.
+  health_check_ip_pool_.push_back("74.125.224.47");
+  health_check_ip_pool_.push_back("74.125.224.79");
+  health_check_ip_pool_.push_back("74.125.224.111");
+  health_check_ip_pool_.push_back("74.125.224.143");
+
   LOG(INFO) << "Device created: " << link_name_
             << " index " << interface_index_;
 }
@@ -534,6 +546,26 @@
                                  link_name_,
                                  technology_,
                                  manager_->device_info());
+    set_health_checker(
+        new ConnectionHealthChecker(
+            connection_,
+            dispatcher_,
+            Bind(&Device::OnConnectionHealthCheckerResult,
+                 weak_ptr_factory_.GetWeakPtr())));
+    InitializeHealthCheckIps();
+  }
+}
+
+void Device::InitializeHealthCheckIps() {
+  IPAddress ip(IPAddress::kFamilyIPv4);
+  for (size_t i = 0; i < health_check_ip_pool_.size(); ++i) {
+    // TODO(armansito): DNS resolution fails when we run out-of-credit
+    // and shill currently doesn't cache resolved IPs. The hardcoded IP
+    // here corresponds to gstatic.google.com/generate_204, however I do
+    // not know if it will remain static. We should implement some sort of
+    // caching after portal detection for this to work.
+    ip.SetAddressFromString(health_check_ip_pool_[i]);
+    health_checker_->AddRemoteIP(ip);
   }
 }
 
@@ -546,6 +578,7 @@
     selected_service_->SetConnection(NULL);
   }
   connection_ = NULL;
+  health_checker_.reset();
 }
 
 void Device::SelectService(const ServiceRefPtr &service) {
@@ -622,6 +655,25 @@
   manager_->UpdateDevice(this);
 }
 
+void Device::RequestConnectionHealthCheck() {
+  if (!health_checker_.get()) {
+    SLOG(Device, 2) << "No health checker exists, cannot request "
+                    << "health check.";
+    return;
+  }
+  if (health_checker_->health_check_in_progress()) {
+    SLOG(Device, 2) << "Health check already in progress.";
+    return;
+  }
+  health_checker_->Start();
+}
+
+void Device::OnConnectionHealthCheckerResult(
+    ConnectionHealthChecker::Result result) {
+  SLOG(Device, 2) << FriendlyName()
+      << ": ConnectionHealthChecker result: " << result;
+}
+
 bool Device::RestartPortalDetection() {
   StopPortalDetection();
   return StartPortalDetection();
@@ -752,6 +804,10 @@
   traffic_monitor_.reset(traffic_monitor);
 }
 
+void Device::set_health_checker(ConnectionHealthChecker *health_checker) {
+  health_checker_.reset(health_checker);
+}
+
 bool Device::StartTrafficMonitor() {
   SLOG(Device, 2) << __func__;
   if (!traffic_monitor_enabled_) {
@@ -781,7 +837,7 @@
 
 void Device::OnNoNetworkRouting() {
   SLOG(Device, 2) << "Device " << FriendlyName()
-                  << ": Traffic Monitor detects there is no incoming traffic.";
+                  << ": Traffic Monitor detects network congestion.";
 }
 
 void Device::SetServiceConnectedState(Service::ConnectState state) {
diff --git a/device.h b/device.h
index fa1ec80..c173152 100644
--- a/device.h
+++ b/device.h
@@ -16,6 +16,7 @@
 
 #include "shill/adaptor_interfaces.h"
 #include "shill/callbacks.h"
+#include "shill/connection_health_checker.h"
 #include "shill/event_dispatcher.h"
 #include "shill/ip_address.h"
 #include "shill/ipconfig.h"
@@ -214,11 +215,15 @@
 
  protected:
   friend class base::RefCounted<Device>;
+  friend class DeviceHealthCheckerTest;
   FRIEND_TEST(CellularServiceTest, IsAutoConnectable);
   FRIEND_TEST(CellularTest, EnableTrafficMonitor);
   FRIEND_TEST(CellularTest, ModemStateChangeDisable);
   FRIEND_TEST(CellularTest, UseNoArpGateway);
   FRIEND_TEST(ConnectionTest, OnRouteQueryResponse);
+  FRIEND_TEST(DeviceHealthCheckerTest, CreateConnectionCreatesHealthChecker);
+  FRIEND_TEST(DeviceHealthCheckerTest, InitializeHealthCheckIps);
+  FRIEND_TEST(DeviceHealthCheckerTest, RequestConnectionHealthCheck);
   FRIEND_TEST(DeviceTest, AcquireIPConfig);
   FRIEND_TEST(DeviceTest, DestroyIPConfig);
   FRIEND_TEST(DeviceTest, DestroyIPConfigNULL);
@@ -330,6 +335,18 @@
   // Avoids showing a failure mole in the UI.
   void SetServiceFailureSilent(Service::ConnectFailure failure_state);
 
+  // Initializes the health checker's internal collection of remote IPs to try.
+  void InitializeHealthCheckIps();
+
+  // Checks the network connectivity status by creating a TCP connection, and
+  // optionally sending a small amout of data.
+  void RequestConnectionHealthCheck();
+
+  // Responds to the result from connection health checker in a device specific
+  // manner.
+  virtual void OnConnectionHealthCheckerResult(
+      ConnectionHealthChecker::Result result);
+
   // Called by the Portal Detector whenever a trial completes.  Device
   // subclasses that choose unique mappings from portal results to connected
   // states can override this method in order to do so.
@@ -407,6 +424,9 @@
     traffic_monitor_enabled_ = traffic_monitor_enabled;
   }
 
+  // Ownership of |health_checker_| is taken.
+  void set_health_checker(ConnectionHealthChecker *health_checker);
+
  private:
   friend class CellularCapabilityTest;
   friend class CellularTest;
@@ -510,6 +530,7 @@
   scoped_ptr<PortalDetector> portal_detector_;
   scoped_ptr<LinkMonitor> link_monitor_;
   scoped_ptr<TrafficMonitor> traffic_monitor_;
+  scoped_ptr<ConnectionHealthChecker> health_checker_;
   bool traffic_monitor_enabled_;
   base::Callback<void(const PortalDetector::Result &)>
       portal_detector_callback_;
@@ -531,6 +552,10 @@
   DHCPProvider *dhcp_provider_;
   RTNLHandler *rtnl_handler_;
 
+  // TODO(armansito): List of static IPs for connection health check.
+  // These should be removed once shill has a DNS result caching mechanism.
+  std::vector<std::string> health_check_ip_pool_;
+
   DISALLOW_COPY_AND_ASSIGN(Device);
 };
 
diff --git a/device_unittest.cc b/device_unittest.cc
index a2d65bf..c5cd166 100644
--- a/device_unittest.cc
+++ b/device_unittest.cc
@@ -12,6 +12,10 @@
 #include <string>
 #include <vector>
 
+#include <base/basictypes.h>
+#include <base/bind.h>
+#include <base/callback.h>
+#include <base/memory/weak_ptr.h>
 #include <chromeos/dbus/service_constants.h>
 #include <dbus-c++/dbus.h>
 #include <gmock/gmock.h>
@@ -22,6 +26,7 @@
 #include "shill/event_dispatcher.h"
 #include "shill/mock_adaptors.h"
 #include "shill/mock_connection.h"
+#include "shill/mock_connection_health_checker.h"
 #include "shill/mock_control.h"
 #include "shill/mock_device.h"
 #include "shill/mock_device_info.h"
@@ -43,12 +48,15 @@
 #include "shill/technology.h"
 #include "shill/traffic_monitor.h"
 
+using base::Bind;
+using base::Callback;
 using std::map;
 using std::string;
 using std::vector;
 using ::testing::_;
 using ::testing::AnyNumber;
 using ::testing::AtLeast;
+using ::testing::DefaultValue;
 using ::testing::Invoke;
 using ::testing::Mock;
 using ::testing::NiceMock;
@@ -1073,4 +1081,84 @@
   EXPECT_TRUE(ExpectByteCounts(device, 0, 0));
 }
 
+class DeviceHealthCheckerTest : public DeviceTest {
+ public:
+  DeviceHealthCheckerTest()
+      : connection_(new StrictMock<MockConnection>(&device_info_)),
+        weak_ptr_factory_(this) {}
+
+ protected:
+  void SetUp() {
+    DeviceTest::SetUp();
+
+    string default_str;
+    vector<string> default_vector;
+
+    ON_CALL(*connection_.get(), interface_name())
+        .WillByDefault(ReturnRef(default_str));
+    ON_CALL(*connection_.get(), dns_servers())
+        .WillByDefault(ReturnRef(default_vector));
+    EXPECT_CALL(*connection_.get(), interface_name())
+        .Times(AnyNumber());
+    EXPECT_CALL(*connection_.get(), dns_servers())
+        .Times(AnyNumber());
+
+    mock_health_checker_.reset(
+        new MockConnectionHealthChecker(
+            connection_,
+            NULL,
+            Bind(&DeviceHealthCheckerTest::OnConnectionHealthCheckerResult,
+                 weak_ptr_factory_.GetWeakPtr())));
+  }
+
+  void SetMockHealthChecker() {
+    device_->set_health_checker(mock_health_checker_.release());
+    EXPECT_TRUE(device_->health_checker_.get());
+  }
+
+  void OnConnectionHealthCheckerResult(
+    ConnectionHealthChecker::Result result) {
+  }
+
+  scoped_refptr<MockConnection> connection_;
+  scoped_ptr<MockConnectionHealthChecker> mock_health_checker_;
+  base::WeakPtrFactory<DeviceHealthCheckerTest> weak_ptr_factory_;
+};
+
+TEST_F(DeviceHealthCheckerTest, CreateConnectionCreatesHealthChecker) {
+  EXPECT_FALSE(device_->connection_.get());
+  EXPECT_FALSE(device_->health_checker_.get());
+  device_->CreateConnection();
+  EXPECT_TRUE(device_->connection_.get());
+  EXPECT_TRUE(device_->health_checker_.get());
+}
+
+TEST_F(DeviceHealthCheckerTest, InitializeHealthCheckIps) {
+  EXPECT_FALSE(device_->health_checker_.get());
+  MockConnectionHealthChecker *health_checker = mock_health_checker_.get();
+  SetMockHealthChecker();
+  EXPECT_CALL(*health_checker, AddRemoteIP(_)).Times(
+      device_->health_check_ip_pool_.size());
+  device_->InitializeHealthCheckIps();
+}
+
+TEST_F(DeviceHealthCheckerTest, RequestConnectionHealthCheck) {
+  EXPECT_FALSE(device_->health_checker_.get());
+  MockConnectionHealthChecker *health_checker = mock_health_checker_.get();
+  SetMockHealthChecker();
+  EXPECT_CALL(*health_checker, Start()).Times(0);
+  EXPECT_CALL(*health_checker, health_check_in_progress())
+      .Times(1)
+      .WillOnce(Return(true));
+  device_->RequestConnectionHealthCheck();
+  Mock::VerifyAndClearExpectations(health_checker);
+
+  EXPECT_CALL(*health_checker, Start()).Times(1);
+  EXPECT_CALL(*health_checker, health_check_in_progress())
+      .Times(1)
+      .WillOnce(Return(false));
+  device_->RequestConnectionHealthCheck();
+  Mock::VerifyAndClearExpectations(health_checker);
+}
+
 }  // namespace shill
diff --git a/mock_cellular_service.h b/mock_cellular_service.h
index afc7020..5debe1b 100644
--- a/mock_cellular_service.h
+++ b/mock_cellular_service.h
@@ -26,6 +26,7 @@
   MOCK_METHOD1(SetLastGoodApn, void(const Stringmap &apn_info));
   MOCK_METHOD0(ClearLastGoodApn, void());
   MOCK_METHOD1(SetActivationState, void(const std::string &state));
+  MOCK_METHOD1(Disconnect, void(Error *error));
 
  private:
   DISALLOW_COPY_AND_ASSIGN(MockCellularService);
diff --git a/mock_connection_health_checker.cc b/mock_connection_health_checker.cc
new file mode 100644
index 0000000..cda845f
--- /dev/null
+++ b/mock_connection_health_checker.cc
@@ -0,0 +1,22 @@
+// Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "shill/mock_connection_health_checker.h"
+
+#include "shill/async_connection.h"
+#include "shill/connection.h"
+
+using base::Callback;
+
+namespace shill {
+
+MockConnectionHealthChecker::MockConnectionHealthChecker(
+    ConnectionRefPtr connection,
+    EventDispatcher *dispatcher,
+    const Callback<void(Result)> &result_callback)
+    : ConnectionHealthChecker(connection, dispatcher, result_callback) {}
+
+MockConnectionHealthChecker::~MockConnectionHealthChecker() {}
+
+}  // namespace shill
diff --git a/mock_connection_health_checker.h b/mock_connection_health_checker.h
new file mode 100644
index 0000000..03d71a1
--- /dev/null
+++ b/mock_connection_health_checker.h
@@ -0,0 +1,34 @@
+// Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SHILL_MOCK_CONNECTION_HEALTH_CHECKER_H_
+#define SHILL_MOCK_CONNECTION_HEALTH_CHECKER_H_
+
+#include <gmock/gmock.h>
+
+#include "shill/connection_health_checker.h"
+
+namespace shill {
+
+class MockConnectionHealthChecker : public ConnectionHealthChecker {
+ public:
+  MockConnectionHealthChecker(
+      ConnectionRefPtr connection,
+      EventDispatcher *dispatcher,
+      const base::Callback<void(Result)> &result_callback);
+  virtual ~MockConnectionHealthChecker();
+
+  MOCK_METHOD1(AddRemoteURL, void(const std::string &url_string));
+  MOCK_METHOD1(AddRemoteIP, void(IPAddress ip));
+  MOCK_METHOD0(Start, void());
+  MOCK_METHOD0(Stop, void());
+  MOCK_CONST_METHOD0(health_check_in_progress, bool());
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(MockConnectionHealthChecker);
+};
+
+}  // namespace shill
+
+#endif  // SHILL_MOCK_CONNECTION_HEALTH_CHECKER_H_
diff --git a/traffic_monitor.cc b/traffic_monitor.cc
index 84717a3..6d6fd76 100644
--- a/traffic_monitor.cc
+++ b/traffic_monitor.cc
@@ -107,6 +107,10 @@
       if (curr_tx_queue_it == curr_tx_queue_lengths.end() ||
           curr_tx_queue_it->second < old_tx_queue_it->second) {
         congested_tx_queues = false;
+        // TODO(armansito): If we had a false positive earlier, we may
+        // want to correct it here by invoking a "connection back to normal
+        // callback", so that the OutOfCredits property can be set to
+        // false.
         break;
       }
     }