shill: Spawn mulitple DNS queries when a remote URL is added to ConnectionHealthChecker

When ConnectionHealthChecker::AddRemoteURL is called,
     (1) Multiple DNS queries are made for the same URL.
     (2) All non-duplicate IP addresses returned are stored.
(3) When trying to connection to remote hosts, IP addresses are chosen
at random.

BUG=chromium:229985
TEST=run and build unit-tests

Change-Id: I3ac062bc27071ea13606ce59e0f4a2313e64716e
Reviewed-on: https://gerrit.chromium.org/gerrit/47945
Commit-Queue: Ben Chan <benchan@chromium.org>
Reviewed-by: Ben Chan <benchan@chromium.org>
Tested-by: Ben Chan <benchan@chromium.org>
diff --git a/Makefile b/Makefile
index 4692789..55769e3 100644
--- a/Makefile
+++ b/Makefile
@@ -225,6 +225,7 @@
 	dhcpcd_proxy.o \
 	diagnostics_reporter.o \
 	dns_client.o \
+	dns_client_factory.o \
 	eap_credentials.o \
 	eap_listener.o \
 	endpoint.o \
@@ -388,6 +389,7 @@
 	mock_dhcp_proxy.o \
 	mock_diagnostics_reporter.o \
 	mock_dns_client.o \
+	mock_dns_client_factory.o \
 	mock_eap_credentials.o \
 	mock_eap_listener.o \
 	mock_ethernet.o \
diff --git a/connection_health_checker.cc b/connection_health_checker.cc
index ee635ee..52ba738 100644
--- a/connection_health_checker.cc
+++ b/connection_health_checker.cc
@@ -6,8 +6,11 @@
 
 #include <arpa/inet.h>
 #include <netinet/in.h>
+#include <stdlib.h>
 #include <sys/socket.h>
 #include <sys/types.h>
+#include <time.h>
+
 #include <vector>
 
 #include <base/bind.h>
@@ -15,6 +18,7 @@
 #include "shill/async_connection.h"
 #include "shill/connection.h"
 #include "shill/dns_client.h"
+#include "shill/dns_client_factory.h"
 #include "shill/error.h"
 #include "shill/http_url.h"
 #include "shill/ip_address.h"
@@ -29,8 +33,13 @@
 
 namespace shill {
 
-const int ConnectionHealthChecker::kDNSTimeoutSeconds = 5;
+//static
+const int ConnectionHealthChecker::kDNSTimeoutMilliseconds = 5000;
+//static
 const int ConnectionHealthChecker::kMaxConnectionAttempts = 3;
+//static
+const int ConnectionHealthChecker::kNumDNSQueries = 5;
+//static
 const uint16 ConnectionHealthChecker::kRemotePort = 80;
 
 ConnectionHealthChecker::ConnectionHealthChecker(
@@ -52,12 +61,7 @@
                                           dispatcher_,
                                           socket_.get(),
                                           connection_complete_callback_)),
-      dns_client_(new DNSClient(IPAddress::kFamilyIPv4,
-                                connection->interface_name(),
-                                connection->dns_servers(),
-                                kDNSTimeoutSeconds * 1000,
-                                dispatcher,
-                                dns_client_callback_)),
+      dns_client_factory_(DNSClientFactory::GetInstance()),
       run_data_test_(true),
       health_check_in_progress_(false),
       num_connection_attempts_(0) {}
@@ -67,10 +71,12 @@
 }
 
 void ConnectionHealthChecker::AddRemoteIP(IPAddress ip) {
-  remote_ips_.push(ip);
+  remote_ips_.push_back(ip);
 }
 
 void ConnectionHealthChecker::AddRemoteURL(const string &url_string) {
+  GarbageCollectDNSClients();
+
   HTTPURL url;
   if (!url.ParseFromString(url_string)) {
     SLOG(Connection, 2) << __func__ << ": Malformed url: " << url_string << ".";
@@ -81,10 +87,21 @@
                         << " to port 80, requested " << url.port() << ".";
     return;
   }
-  Error error;
-  if (!dns_client_->Start(url.host(), &error)) {
-    SLOG(Connection, 2) << __func__ << ": Failed to start DNS client: "
-                        << error.message();
+  for (int i = 0; i < kNumDNSQueries; ++i) {
+    Error error;
+    DNSClient *dns_client =
+      dns_client_factory_->CreateDNSClient(IPAddress::kFamilyIPv4,
+                                           connection_->interface_name(),
+                                           connection_->dns_servers(),
+                                           kDNSTimeoutMilliseconds,
+                                           dispatcher_,
+                                           dns_client_callback_);
+    dns_clients_.push_back(dns_client);
+    if (!dns_clients_[i]->Start(url.host(), &error)) {
+      SLOG(Connection, 2) << __func__ << ": Failed to start DNS client "
+                          << "(query #" << i << "): "
+                          << error.message();
+    }
   }
 }
 
@@ -140,14 +157,18 @@
 }
 
 void ConnectionHealthChecker::SetupTcpConnection() {
-  IPAddress ip = remote_ips_.front();
+  // Pick a random IP from the set of IPs.
+  // This guards against
+  //   (1) Repeated failed attempts for the same IP at start-up everytime.
+  //   (2) All users attempting to connect to the same IP.
+  int next_ip_index = rand() % remote_ips_.size();
+  const IPAddress &ip = remote_ips_[next_ip_index];
   SLOG(Connection, 3) << __func__ << ": Starting connection at "
                       << ip.ToString();
   if (tcp_connection_->Start(ip, kRemotePort)) {
     // TCP connection successful, no need to try more.
     return;
   }
-
   SLOG(Connection, 2) << __func__ << ": Connection attempt failed.";
   TryNextIP();
 }
@@ -183,15 +204,17 @@
                         << error.message();
     return;
   }
-  remote_ips_.push(ip);
+  // Insert ip into the list of cached IP addresses, if not already present.
+  for (IPAddresses::size_type i = 0; i < remote_ips_.size(); ++i)
+    if (remote_ips_[i].Equals(ip))
+      return;
+  remote_ips_.push_back(ip);
 }
 
 void ConnectionHealthChecker::TryNextIP() {
   ++num_connection_attempts_;
   // Check if enough attempts have been made already.
-  if (num_connection_attempts_ >= kMaxConnectionAttempts ||
-      static_cast<IPAddressQueue::size_type>(num_connection_attempts_)
-           >= remote_ips_.size()) {
+  if (num_connection_attempts_ >= kMaxConnectionAttempts) {
     LOG(INFO) << __func__
               << ": multiple failed attempts to established a TCP connection.";
     // Give up. Clean up and notify client.
@@ -199,10 +222,6 @@
     result_callback_.Run(kResultConnectionFailure);
     return;
   }
-  IPAddress recycle_addr = remote_ips_.front();
-  remote_ips_.pop();
-  remote_ips_.push(recycle_addr);
-
   SetupTcpConnection();
 }
 
@@ -338,4 +357,18 @@
   return false;
 }
 
+void ConnectionHealthChecker::GarbageCollectDNSClients() {
+  ScopedVector<DNSClient> keep;
+  ScopedVector<DNSClient> discard;
+  for (size_t i = 0; i < dns_clients_.size(); ++i) {
+    if (dns_clients_[i]->IsActive())
+      keep.push_back(dns_clients_[i]);
+    else
+      discard.push_back(dns_clients_[i]);
+  }
+  dns_clients_.weak_clear();
+  dns_clients_ = keep.Pass();  // Passes ownership of contents.
+  discard.clear();
+}
+
 }  // namespace shill
diff --git a/connection_health_checker.h b/connection_health_checker.h
index 22ace95..20fca7a 100644
--- a/connection_health_checker.h
+++ b/connection_health_checker.h
@@ -5,7 +5,7 @@
 #ifndef SHILL_CONNECTION_HEALTH_CHECKER_H_
 #define SHILL_CONNECTION_HEALTH_CHECKER_H_
 
-#include <queue>
+#include <vector>
 
 #include <base/basictypes.h>
 #include <base/callback.h>
@@ -22,6 +22,7 @@
 
 class AsyncConnection;
 class DNSClient;
+class DNSClientFactory;
 class Error;
 class EventDispatcher;
 class IPAddress;
@@ -37,7 +38,7 @@
 //   -(3)- Connectivity OK (TCP connection established, is healthy)
 class ConnectionHealthChecker {
  public:
-  typedef std::queue<IPAddress> IPAddressQueue;
+  typedef std::vector<IPAddress> IPAddresses;
 
   // TODO(pprabhu): Rename kResultElongatedTimeWait to kResultTearDownFailure.
   enum Result {
@@ -88,7 +89,7 @@
   static const char *ResultToString(Result result);
 
   // Accessors.
-  const IPAddressQueue &remote_ips() { return remote_ips_; }
+  const IPAddresses &remote_ips() const { return remote_ips_; }
   void set_run_data_test(bool val) { run_data_test_ = val; }
   virtual bool health_check_in_progress() const {
       return health_check_in_progress_;
@@ -96,14 +97,17 @@
 
  private:
   friend class ConnectionHealthCheckerTest;
+  FRIEND_TEST(ConnectionHealthCheckerTest, GarbageCollectDNSClients);
   FRIEND_TEST(ConnectionHealthCheckerTest, GetSocketInfo);
   FRIEND_TEST(ConnectionHealthCheckerTest, SendData);
   FRIEND_TEST(ConnectionHealthCheckerTest, ShutDown);
 
   // Time to wait for DNS server.
-  static const int kDNSTimeoutSeconds;
+  static const int kDNSTimeoutMilliseconds;
   // Number of connection attempts before failure per health check request.
   static const int kMaxConnectionAttempts;
+  // Number of DNS queries to be spawned when a new remote URL is added.
+  static const int kNumDNSQueries;
   static const uint16 kRemotePort;
 
   // Start a new AsyncConnection with callback set to OnConnectionComplete().
@@ -120,12 +124,13 @@
   Result SendData(int sock_fd);
   Result ShutDown(int sock_fd);
   bool GetSocketInfo(int sock_fd, SocketInfo *sock_info);
+  void GarbageCollectDNSClients();
 
   ConnectionRefPtr connection_;
   EventDispatcher *dispatcher_;
   base::Callback<void(Result)> result_callback_;
 
-  IPAddressQueue remote_ips_;
+  IPAddresses remote_ips_;
   scoped_ptr<SocketInfoReader> socket_info_reader_;
   scoped_ptr<Sockets> socket_;
   base::WeakPtrFactory<ConnectionHealthChecker> weak_ptr_factory_;
@@ -133,7 +138,8 @@
   const base::Callback<void(const Error&, const IPAddress&)>
       dns_client_callback_;
   scoped_ptr<AsyncConnection> tcp_connection_;
-  scoped_ptr<DNSClient> dns_client_;
+  DNSClientFactory *dns_client_factory_;
+  ScopedVector<DNSClient> dns_clients_;
   // If true, HealthChecker attempts to send a small amount of data over
   // the network during the test. Otherwise, the inference is based on
   // the connection open/close behaviour.
diff --git a/connection_health_checker_unittest.cc b/connection_health_checker_unittest.cc
index df382ca..5055a87 100644
--- a/connection_health_checker_unittest.cc
+++ b/connection_health_checker_unittest.cc
@@ -11,12 +11,14 @@
 #include <base/bind.h>
 #include <base/callback.h>
 #include <base/memory/scoped_ptr.h>
+#include <base/memory/scoped_vector.h>
 #include <gtest/gtest.h>
 
 #include "shill/mock_async_connection.h"
 #include "shill/mock_connection.h"
 #include "shill/mock_control.h"
 #include "shill/mock_dns_client.h"
+#include "shill/mock_dns_client_factory.h"
 #include "shill/mock_device_info.h"
 #include "shill/mock_event_dispatcher.h"
 #include "shill/mock_sockets.h"
@@ -66,7 +68,7 @@
         device_info_(&control_, &dispatcher_,
                      reinterpret_cast<Metrics*>(NULL),
                      reinterpret_cast<Manager*>(NULL)),
-        connection_(new MockConnection(&device_info_)),
+        connection_(new NiceMock<MockConnection>(&device_info_)),
         socket_(NULL) {}
 
   // Invokes
@@ -99,12 +101,23 @@
     health_checker_->OnConnectionComplete(success, sock_fd);
   }
 
+  void InvokeGetDNSResultFailure() {
+    Error error(Error::kOperationFailed, "");
+    IPAddress address(IPAddress::kFamilyUnknown);
+    health_checker_->GetDNSResult(error, address);
+  }
+
+  void InvokeGetDNSResultSuccess(const IPAddress &address) {
+    Error error;
+    health_checker_->GetDNSResult(error, address);
+  }
+
  protected:
   void SetUp() {
     EXPECT_CALL(*connection_.get(), interface_name())
         .WillRepeatedly(ReturnRef(interface_name_));
-    EXPECT_CALL(*connection_.get(), dns_servers())
-        .WillOnce(ReturnRef(dns_servers_));
+    ON_CALL(*connection_.get(), dns_servers())
+        .WillByDefault(ReturnRef(dns_servers_));
     health_checker_.reset(
         new ConnectionHealthChecker(
              connection_,
@@ -115,12 +128,11 @@
     socket_ = new StrictMock<MockSockets>();
     tcp_connection_ = new StrictMock<MockAsyncConnection>();
     socket_info_reader_ = new StrictMock<MockSocketInfoReader>();
-    dns_client_ = new MockDNSClient();
     // Passes ownership for all of these.
     health_checker_->socket_.reset(socket_);
     health_checker_->tcp_connection_.reset(tcp_connection_);
     health_checker_->socket_info_reader_.reset(socket_info_reader_);
-    health_checker_->dns_client_.reset(dns_client_);
+    health_checker_->dns_client_factory_ = MockDNSClientFactory::GetInstance();
   }
 
   void TearDown() {
@@ -133,26 +145,31 @@
     return health_checker_->socket_.get();
   }
   const AsyncConnection *tcp_connection() {
-    return health_checker_->tcp_connection_.get(); }
+    return health_checker_->tcp_connection_.get();
+  }
+  ScopedVector<DNSClient> &dns_clients() {
+    return health_checker_->dns_clients_;
+  }
   bool health_check_in_progress() {
     return health_checker_->health_check_in_progress_;
   }
   short num_connection_attempts() {
     return health_checker_->num_connection_attempts_;
   }
-  const ConnectionHealthChecker::IPAddressQueue &remote_ips() {
+  ConnectionHealthChecker::IPAddresses &remote_ips() {
     return health_checker_->remote_ips_;
   }
   int MaxConnectionAttempts() {
     return ConnectionHealthChecker::kMaxConnectionAttempts;
   }
+  int NumDNSQueries() {
+    return ConnectionHealthChecker::kNumDNSQueries;
+  }
 
   // Mock Callbacks
   MOCK_METHOD1(ResultCallbackTarget,
                void(ConnectionHealthChecker::Result result));
 
-
-
   // Helper methods
   IPAddress StringToIPv4Address(const string &address_string) {
     IPAddress ip_address(IPAddress::kFamilyIPv4);
@@ -198,15 +215,6 @@
         0,
         SocketInfo::kTimerStateUnknown);
   }
-  void GetDNSResultFailure() {
-    Error error(Error::kOperationFailed, "");
-    IPAddress address(IPAddress::kFamilyUnknown);
-    health_checker_->GetDNSResult(error, address);
-  }
-  void GetDNSResultSuccess(const IPAddress &address) {
-    Error error;
-    health_checker_->GetDNSResult(error, address);
-  }
   void VerifyAndClearAllExpectations() {
     Mock::VerifyAndClearExpectations(this);
     Mock::VerifyAndClearExpectations(tcp_connection_);
@@ -338,13 +346,12 @@
   NiceMock<MockDeviceInfo> device_info_;
   vector<string> dns_servers_;
 
-  scoped_refptr<MockConnection> connection_;
+  scoped_refptr<NiceMock<MockConnection>> connection_;
   StrictMock<MockEventDispatcher> dispatcher_;
   StrictMock<MockSockets> *socket_;
   StrictMock<MockSocketInfoReader> *socket_info_reader_;
   StrictMock<MockAsyncConnection> *tcp_connection_;
-  MockDNSClient *dns_client_;
-  // Exepctations inthe Expect* functions are put in this sequence.
+  // Expectations in the Expect* functions are put in this sequence.
   // This allows us to chain calls to Expect* functions.
   Sequence seq_;
 
@@ -361,43 +368,125 @@
   IPAddress ip = StringToIPv4Address(kIPAddress_0_0_0_0);
   health_checker_->AddRemoteIP(ip);
   EXPECT_EQ(1, remote_ips().size());
-  EXPECT_TRUE(remote_ips().front().Equals(ip));
+  EXPECT_TRUE(remote_ips().rbegin()->Equals(ip));
 
   health_checker_->AddRemoteIP(
       StringToIPv4Address(kIPAddress_0_0_0_0));
   EXPECT_EQ(2, remote_ips().size());
 }
 
+TEST_F(ConnectionHealthCheckerTest, GarbageCollectDNSClients) {
+  dns_clients().clear();
+  health_checker_->GarbageCollectDNSClients();
+  EXPECT_TRUE(dns_clients().empty());
+
+  for (int i = 0; i < 3; ++i) {
+    MockDNSClient *dns_client = new MockDNSClient();
+    EXPECT_CALL(*dns_client, IsActive())
+        .WillOnce(Return(true))
+        .WillOnce(Return(true))
+        .WillOnce(Return(false));
+    // Takes ownership.
+    dns_clients().push_back(dns_client);
+  }
+  for (int i = 0; i < 2; ++i) {
+    MockDNSClient *dns_client = new MockDNSClient();
+    EXPECT_CALL(*dns_client, IsActive())
+        .WillOnce(Return(false));
+    // Takes ownership.
+    dns_clients().push_back(dns_client);
+  }
+
+  EXPECT_EQ(5, dns_clients().size());
+  health_checker_->GarbageCollectDNSClients();
+  EXPECT_EQ(3, dns_clients().size());
+  health_checker_->GarbageCollectDNSClients();
+  EXPECT_EQ(3, dns_clients().size());
+  health_checker_->GarbageCollectDNSClients();
+  EXPECT_TRUE(dns_clients().empty());
+}
+
 TEST_F(ConnectionHealthCheckerTest, AddRemoteURL) {
-  ConnectionHealthChecker::IPAddressQueue::size_type num_old_ips;
-
-  // DNS query fails synchronously.
-  EXPECT_CALL(*dns_client_, Start(_,_))
-      .WillOnce(Return(false));
-  num_old_ips = remote_ips().size();
-  health_checker_->AddRemoteURL(kProxyURLRemote);
-  EXPECT_EQ(num_old_ips, remote_ips().size());
-  Mock::VerifyAndClearExpectations(dns_client_);
-
-  // DNS query fails asynchronously.
-  EXPECT_CALL(*dns_client_, Start(_,_))
-      .WillOnce(Return(true));
-  num_old_ips = remote_ips().size();
-  health_checker_->AddRemoteURL(kProxyURLRemote);
-  GetDNSResultFailure();
-  EXPECT_EQ(num_old_ips, remote_ips().size());
-  Mock::VerifyAndClearExpectations(dns_client_);
-
-  // Success
-  EXPECT_CALL(*dns_client_, Start(_,_))
-      .WillOnce(Return(true));
-  num_old_ips = remote_ips().size();
-  health_checker_->AddRemoteURL(kProxyURLRemote);
+  HTTPURL url;
+  url.ParseFromString(kProxyURLRemote);
+  string host = url.host();
   IPAddress remote_ip = StringToIPv4Address(kProxyIPAddressRemote);
-  GetDNSResultSuccess(remote_ip);
+  IPAddress remote_ip_2 = StringToIPv4Address(kIPAddress_8_8_8_8);
+
+  MockDNSClientFactory *dns_client_factory
+      = MockDNSClientFactory::GetInstance();
+  ConnectionHealthChecker::IPAddresses::size_type num_old_ips;
+  vector<MockDNSClient *> dns_client_buffer;
+
+  // All DNS queries fail.
+  for (int i = 0; i < NumDNSQueries(); ++i) {
+    MockDNSClient *dns_client = new MockDNSClient();
+    EXPECT_CALL(*dns_client, Start(host, _))
+        .WillOnce(Return(false));
+    dns_client_buffer.push_back(dns_client);
+  }
+  // Will pass ownership of dns_clients elements.
+  for (int i = 0; i < NumDNSQueries(); ++i) {
+    EXPECT_CALL(*dns_client_factory, CreateDNSClient(_,_,_,_,_,_))
+        .InSequence(seq_)
+        .WillOnce(Return(dns_client_buffer[i]));
+  }
+  num_old_ips = remote_ips().size();
+  health_checker_->AddRemoteURL(kProxyURLRemote);
+  EXPECT_EQ(num_old_ips, remote_ips().size());
+  Mock::VerifyAndClearExpectations(dns_client_factory);
+  dns_client_buffer.clear();
+  dns_clients().clear();
+
+  // All but one DNS queries fail, 1 succeeds.
+  for (int i = 0; i < NumDNSQueries(); ++i) {
+    MockDNSClient *dns_client = new MockDNSClient();
+    EXPECT_CALL(*dns_client, Start(host, _))
+        .WillOnce(Return(true));
+    dns_client_buffer.push_back(dns_client);
+  }
+  // Will pass ownership of dns_clients elements.
+  for (int i = 0; i < NumDNSQueries(); ++i) {
+    EXPECT_CALL(*dns_client_factory, CreateDNSClient(_,_,_,_,_,_))
+        .InSequence(seq_)
+        .WillOnce(Return(dns_client_buffer[i]));
+  }
+  num_old_ips = remote_ips().size();
+  health_checker_->AddRemoteURL(kProxyURLRemote);
+  for(int i = 0; i < NumDNSQueries() - 1; ++i) {
+    InvokeGetDNSResultFailure();
+  }
+  InvokeGetDNSResultSuccess(remote_ip);
   EXPECT_EQ(num_old_ips + 1, remote_ips().size());
-  EXPECT_TRUE(remote_ip.Equals(remote_ips().front()));
-  Mock::VerifyAndClearExpectations(dns_client_);
+  EXPECT_TRUE(remote_ip.Equals(*remote_ips().rbegin()));
+  Mock::VerifyAndClearExpectations(dns_client_factory);
+  dns_client_buffer.clear();
+  dns_clients().clear();
+
+  // Only 2 distinct IP addresses are returned.
+  for (int i = 0; i < NumDNSQueries(); ++i) {
+    MockDNSClient *dns_client = new MockDNSClient();
+    EXPECT_CALL(*dns_client, Start(host, _))
+        .WillOnce(Return(true));
+    dns_client_buffer.push_back(dns_client);
+  }
+  // Will pass ownership of dns_clients elements.
+  for (int i = 0; i < NumDNSQueries(); ++i) {
+    EXPECT_CALL(*dns_client_factory, CreateDNSClient(_,_,_,_,_,_))
+        .InSequence(seq_)
+        .WillOnce(Return(dns_client_buffer[i]));
+  }
+  remote_ips().clear();
+  num_old_ips = remote_ips().size();
+  health_checker_->AddRemoteURL(kProxyURLRemote);
+  for (int i = 0; i < NumDNSQueries() - 1; ++i) {
+    InvokeGetDNSResultSuccess(remote_ip);
+  }
+  InvokeGetDNSResultSuccess(remote_ip_2);
+  EXPECT_EQ(num_old_ips + 2, remote_ips().size());
+  Mock::VerifyAndClearExpectations(dns_client_factory);
+  dns_client_buffer.clear();
+  dns_clients().clear();
 }
 
 TEST_F(ConnectionHealthCheckerTest, GetSocketInfo) {
@@ -792,7 +881,8 @@
   EXPECT_CALL(*tcp_connection_,
               Start(IsSameIPAddress(StringToIPv4Address(kProxyIPAddressRemote)),
                     kProxyPortRemote))
-      .WillOnce(Return(false));
+      .Times(MaxConnectionAttempts())
+      .WillRepeatedly(Return(false));
   EXPECT_CALL(*tcp_connection_, Stop())
       .Times(1);
   EXPECT_CALL(*this, ResultCallbackTarget(
@@ -803,14 +893,39 @@
   Mock::VerifyAndClearExpectations(tcp_connection_);
 }
 
-// Precondition: len(|remote_ips_|) == 1
-// Flow: Start() -> asynchronous async_connection failure
+//
+// Flow: Start() -> Connection Successful
+//               -> Forever(SendData() returns kResultUnknown)
 // Expectation: call |result_callback| with kResultConnectionFailure
-TEST_F(ConnectionHealthCheckerTest, StartAsynchrnousFailure) {
+//              (2) Sockets::Close called |kMaxConnectionAttempts| times
+//
+TEST_F(ConnectionHealthCheckerTest, HealthCheckFailHitMaxConnectionAttempts) {
+  ExpectSuccessfulStart();
+  ExpectSendDataReturns(ConnectionHealthChecker::kResultUnknown);
+  for (int i = 0; i < MaxConnectionAttempts() - 1; ++i) {
+    ExpectRetry();
+    ExpectSendDataReturns(ConnectionHealthChecker::kResultUnknown);
+  }
+  ExpectCleanUp();
+  EXPECT_CALL(*this, ResultCallbackTarget(
+                           ConnectionHealthChecker::kResultConnectionFailure));
+  for (int i = 0; i < MaxConnectionAttempts() + 2; ++i)
+    health_checker_->AddRemoteIP(StringToIPv4Address(kProxyIPAddressRemote));
+  health_checker_->Start();
+  for (int i = 0; i < MaxConnectionAttempts(); ++i)
+    InvokeOnConnectionComplete(true, kProxyFD);
+  VerifyAndClearAllExpectations();
+}
+
+// Precondition: len(|remote_ips_|) == 1
+// Flow: Start() -> forever(asynchronous async_connection failure)
+// Expectation: call |result_callback| with kResultConnectionFailure
+TEST_F(ConnectionHealthCheckerTest, StartAsynchronousFailure) {
   EXPECT_CALL(*tcp_connection_,
               Start(IsSameIPAddress(StringToIPv4Address(kProxyIPAddressRemote)),
                     kProxyPortRemote))
-      .WillOnce(Return(true));
+      .Times(MaxConnectionAttempts())
+      .WillRepeatedly(Return(true));
   EXPECT_CALL(*tcp_connection_, Stop())
       .Times(1);
   EXPECT_CALL(*this, ResultCallbackTarget(
@@ -818,7 +933,8 @@
       .Times(1);
   health_checker_->AddRemoteIP(StringToIPv4Address(kProxyIPAddressRemote));
   health_checker_->Start();
-  InvokeOnConnectionComplete(false, -1);  // second argument ignored.
+  for (int i = 0; i < MaxConnectionAttempts(); ++i)
+    InvokeOnConnectionComplete(false, -1);
   Mock::VerifyAndClearExpectations(this);
   Mock::VerifyAndClearExpectations(tcp_connection_);
 }
@@ -877,30 +993,6 @@
 }
 
 // Precondition: len(|remote_ips_|) == 2
-// Flow: Start() -> Connection Successful ->
-//       SendData() returns kResultUnknown ->
-//       SendData() returns kResultUnknown
-// Expectation: (1) call |result_callback| with kResultConnectionFailure
-//              (2) Sockets::Close called twice
-TEST_F(ConnectionHealthCheckerTest, HealthCheckFailOutOfIPs) {
-  ExpectSuccessfulStart();
-  ExpectSendDataReturns(ConnectionHealthChecker::kResultUnknown);
-  ExpectRetry();
-  ExpectSendDataReturns(ConnectionHealthChecker::kResultUnknown);
-  ExpectCleanUp();
-  // Expectation:
-  EXPECT_CALL(*this, ResultCallbackTarget(
-                           ConnectionHealthChecker::kResultConnectionFailure))
-      .Times(1);
-  health_checker_->AddRemoteIP(StringToIPv4Address(kProxyIPAddressRemote));
-  health_checker_->AddRemoteIP(StringToIPv4Address(kProxyIPAddressRemote));
-  health_checker_->Start();
-  InvokeOnConnectionComplete(true, kProxyFD);
-  InvokeOnConnectionComplete(true, kProxyFD);
-  VerifyAndClearAllExpectations();
-}
-
-// Precondition: len(|remote_ips_|) == 2
 // Flow: Start() -> Connection Successful -> SendData() returns kResultUnknown
 //       -> SendData() returns kResultSuccess
 // Expectation: call |result_callback| with kResultSuccess
@@ -923,31 +1015,4 @@
   VerifyAndClearAllExpectations();
 }
 
-//
-// Precondition: len(|remote_ips_|) > |kMaxConnectionAttempts|
-// Flow: Start() -> Connection Successful
-//               -> Forever(SendData() returns kResultUnknown)
-// Expectation: call |result_callback| with kResultConnectionFailure
-//              (2) Sockets::Close called |kMaxConnectionAttempts| times
-//
-TEST_F(ConnectionHealthCheckerTest, HealthCheckFailHitMaxConnectionAttempts) {
-  ExpectSuccessfulStart();
-  ExpectSendDataReturns(ConnectionHealthChecker::kResultUnknown);
-  for (int i = 0; i < MaxConnectionAttempts()-1; ++i) {
-    ExpectRetry();
-    ExpectSendDataReturns(ConnectionHealthChecker::kResultUnknown);
-  }
-  ExpectCleanUp();
-  // Expectation:
-  EXPECT_CALL(*this, ResultCallbackTarget(
-                           ConnectionHealthChecker::kResultConnectionFailure))
-      .Times(1);
-  for (int i = 0; i < MaxConnectionAttempts() + 2; ++i)
-    health_checker_->AddRemoteIP(StringToIPv4Address(kProxyIPAddressRemote));
-  health_checker_->Start();
-  for (int i = 0; i < MaxConnectionAttempts(); ++i)
-    InvokeOnConnectionComplete(true, kProxyFD);
-  VerifyAndClearAllExpectations();
-}
-
 }  // namespace shill
diff --git a/dns_client.cc b/dns_client.cc
index 9ff1105..b5a93c9 100644
--- a/dns_client.cc
+++ b/dns_client.cc
@@ -151,6 +151,10 @@
   resolver_state_.reset();
 }
 
+bool DNSClient::IsActive() const {
+  return running_;
+}
+
 // We delay our call to completion so that we exit all IOHandlers, and
 // can clean up all of our local state before calling the callback, or
 // during the process of the execution of the callee (which is free to
diff --git a/dns_client.h b/dns_client.h
index dd7afa6..e40e07e 100644
--- a/dns_client.h
+++ b/dns_client.h
@@ -62,6 +62,8 @@
   // invocation.
   virtual void Stop();
 
+  virtual bool IsActive() const;
+
  private:
   friend class DNSClientTest;
 
diff --git a/dns_client_factory.cc b/dns_client_factory.cc
new file mode 100644
index 0000000..3b9b137
--- /dev/null
+++ b/dns_client_factory.cc
@@ -0,0 +1,41 @@
+// Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "shill/dns_client_factory.h"
+
+using std::string;
+using std::vector;
+
+namespace shill {
+
+namespace {
+
+base::LazyInstance<DNSClientFactory> g_dns_client_factory
+    = LAZY_INSTANCE_INITIALIZER;
+
+}  // namespace
+
+DNSClientFactory::DNSClientFactory() {}
+DNSClientFactory::~DNSClientFactory() {}
+
+DNSClientFactory *DNSClientFactory::GetInstance() {
+  return g_dns_client_factory.Pointer();
+}
+
+DNSClient *DNSClientFactory::CreateDNSClient(
+    IPAddress::Family family,
+    const string &interface_name,
+    const vector<string> &dns_servers,
+    int timeout_ms,
+    EventDispatcher *dispatcher,
+    const DNSClient::ClientCallback &callback) {
+  return new DNSClient(family,
+                       interface_name,
+                       dns_servers,
+                       timeout_ms,
+                       dispatcher,
+                       callback);
+}
+
+}  // namespace shill
diff --git a/dns_client_factory.h b/dns_client_factory.h
new file mode 100644
index 0000000..685bbab
--- /dev/null
+++ b/dns_client_factory.h
@@ -0,0 +1,45 @@
+// Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SHILL_DNS_CLIENT_FACTORY_H_
+#define SHILL_DNS_CLIENT_FACTORY_H_
+
+#include <string>
+#include <vector>
+
+#include <base/lazy_instance.h>
+
+#include "shill/dns_client.h"
+#include "shill/event_dispatcher.h"
+#include "shill/ip_address.h"
+
+namespace shill {
+
+class DNSClientFactory {
+ public:
+  virtual ~DNSClientFactory();
+
+  // This is a singleton. Use DNSClientFactory::GetInstance()->Foo()
+  static DNSClientFactory *GetInstance();
+
+  virtual DNSClient *CreateDNSClient(
+      IPAddress::Family family,
+      const std::string &interface_name,
+      const std::vector<std::string> &dns_servers,
+      int timeout_ms,
+      EventDispatcher *dispatcher,
+      const DNSClient::ClientCallback &callback);
+
+ protected:
+  DNSClientFactory();
+
+ private:
+  friend struct base::DefaultLazyInstanceTraits<DNSClientFactory>;
+
+  DISALLOW_COPY_AND_ASSIGN(DNSClientFactory);
+};
+
+}  // namespace shill
+
+#endif  // SHILL_DNS_CLIENT_FACTORY_H_
diff --git a/mock_dns_client.h b/mock_dns_client.h
index e8fcc91..62706e3 100644
--- a/mock_dns_client.h
+++ b/mock_dns_client.h
@@ -19,6 +19,7 @@
 
   MOCK_METHOD2(Start, bool(const std::string &hostname, Error *error));
   MOCK_METHOD0(Stop, void());
+  MOCK_CONST_METHOD0(IsActive, bool());
 
  private:
   DISALLOW_COPY_AND_ASSIGN(MockDNSClient);
diff --git a/mock_dns_client_factory.cc b/mock_dns_client_factory.cc
new file mode 100644
index 0000000..342249c
--- /dev/null
+++ b/mock_dns_client_factory.cc
@@ -0,0 +1,21 @@
+// Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "shill/mock_dns_client_factory.h"
+
+namespace shill {
+
+namespace {
+base::LazyInstance<MockDNSClientFactory> g_mock_dns_client_factory
+    = LAZY_INSTANCE_INITIALIZER;
+}  // namespace
+
+MockDNSClientFactory::MockDNSClientFactory() {}
+MockDNSClientFactory::~MockDNSClientFactory() {}
+
+MockDNSClientFactory *MockDNSClientFactory::GetInstance() {
+  return g_mock_dns_client_factory.Pointer();
+}
+
+}  // namespace shill
diff --git a/mock_dns_client_factory.h b/mock_dns_client_factory.h
new file mode 100644
index 0000000..aa9a541
--- /dev/null
+++ b/mock_dns_client_factory.h
@@ -0,0 +1,46 @@
+// Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SHILL_MOCK_DNS_CLIENT_FACTORY_H_
+#define SHILL_MOCK_DNS_CLIENT_FACTORY_H_
+
+#include <string>
+#include <vector>
+
+#include <base/lazy_instance.h>
+#include <gmock/gmock.h>
+
+#include "shill/dns_client_factory.h"
+#include "shill/event_dispatcher.h"
+#include "shill/ip_address.h"
+
+namespace shill {
+
+class MockDNSClientFactory : public DNSClientFactory {
+ public:
+  virtual ~MockDNSClientFactory();
+
+  // This is a singleton. Use MockDNSClientFactory::GetInstance()->Foo()
+  static MockDNSClientFactory *GetInstance();
+
+  MOCK_METHOD6(CreateDNSClient,
+               DNSClient *(IPAddress::Family family,
+                           const std::string &interface_name,
+                           const std::vector<std::string> &dns_servers,
+                           int timeout_ms,
+                           EventDispatcher *dispatcher,
+                           const DNSClient::ClientCallback &callback));
+
+ protected:
+  MockDNSClientFactory();
+
+ private:
+  friend struct base::DefaultLazyInstanceTraits<MockDNSClientFactory>;
+
+  DISALLOW_COPY_AND_ASSIGN(MockDNSClientFactory);
+};
+
+}  // namespace shill
+
+#endif  // SHILL_MOCK_DNS_CLIENT_FACTORY_H_