diff --git a/Lib/ssl.py b/Lib/ssl.py
index 9e88073..1d29bef 100644
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -115,12 +115,7 @@
                                         cert_reqs, ssl_version, ca_certs,
                                         ciphers)
             if do_handshake_on_connect:
-                timeout = self.gettimeout()
-                try:
-                    self.settimeout(None)
-                    self.do_handshake()
-                finally:
-                    self.settimeout(timeout)
+                self.do_handshake()
         self.keyfile = keyfile
         self.certfile = certfile
         self.cert_reqs = cert_reqs
diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py
index 345a1c7..f75f0b1 100644
--- a/Lib/test/test_poplib.py
+++ b/Lib/test/test_poplib.py
@@ -10,6 +10,7 @@
 import socket
 import os
 import time
+import errno
 
 from unittest import TestCase
 from test import test_support
@@ -231,11 +232,37 @@
         def __init__(self, conn):
             asynchat.async_chat.__init__(self, conn)
             self.socket = ssl.wrap_socket(self.socket, certfile=CERTFILE,
-                                          server_side=True)
+                                          server_side=True,
+                                          do_handshake_on_connect=False)
+            # Must try handshake before calling push()
+            self._ssl_accepting = True
+            self._do_ssl_handshake()
             self.set_terminator("\r\n")
             self.in_buffer = []
             self.push('+OK dummy pop3 server ready.')
 
+        def _do_ssl_handshake(self):
+            try:
+                self.socket.do_handshake()
+            except ssl.SSLError, err:
+                if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
+                                   ssl.SSL_ERROR_WANT_WRITE):
+                    return
+                elif err.args[0] == ssl.SSL_ERROR_EOF:
+                    return self.handle_close()
+                raise
+            except socket.error, err:
+                if err.args[0] == errno.ECONNABORTED:
+                    return self.handle_close()
+            else:
+                self._ssl_accepting = False
+
+        def handle_read(self):
+            if self._ssl_accepting:
+                self._do_ssl_handshake()
+            else:
+                DummyPOP3Handler.handle_read(self)
+
     class TestPOP3_SSLClass(TestPOP3Class):
         # repeat previous tests by using poplib.POP3_SSL
 
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 2c401dc..bbccaeb 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -494,7 +494,8 @@
                     asyncore.dispatcher_with_send.__init__(self, conn)
                     self.socket = ssl.wrap_socket(conn, server_side=True,
                                                   certfile=certfile,
-                                                  do_handshake_on_connect=True)
+                                                  do_handshake_on_connect=False)
+                    self._ssl_accepting = True
 
                 def readable(self):
                     if isinstance(self.socket, ssl.SSLSocket):
@@ -502,9 +503,28 @@
                             self.handle_read_event()
                     return True
 
+                def _do_ssl_handshake(self):
+                    try:
+                        self.socket.do_handshake()
+                    except ssl.SSLError, err:
+                        if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
+                                           ssl.SSL_ERROR_WANT_WRITE):
+                            return
+                        elif err.args[0] == ssl.SSL_ERROR_EOF:
+                            return self.handle_close()
+                        raise
+                    except socket.error, err:
+                        if err.args[0] == errno.ECONNABORTED:
+                            return self.handle_close()
+                    else:
+                        self._ssl_accepting = False
+
                 def handle_read(self):
-                    data = self.recv(1024)
-                    self.send(data.lower())
+                    if self._ssl_accepting:
+                        self._do_ssl_handshake()
+                    else:
+                        data = self.recv(1024)
+                        self.send(data.lower())
 
                 def handle_close(self):
                     self.close()
@@ -1271,6 +1291,53 @@
                 server.stop()
                 server.join()
 
+        def test_handshake_timeout(self):
+            # Issue #5103: SSL handshake must respect the socket timeout
+            server = socket.socket(socket.AF_INET)
+            host = "127.0.0.1"
+            port = test_support.bind_port(server)
+            started = threading.Event()
+            finish = False
+
+            def serve():
+                server.listen(5)
+                started.set()
+                conns = []
+                while not finish:
+                    r, w, e = select.select([server], [], [], 0.1)
+                    if server in r:
+                        # Let the socket hang around rather than having
+                        # it closed by garbage collection.
+                        conns.append(server.accept()[0])
+
+            t = threading.Thread(target=serve)
+            t.start()
+            started.wait()
+
+            try:
+                try:
+                    c = socket.socket(socket.AF_INET)
+                    c.settimeout(0.2)
+                    c.connect((host, port))
+                    # Will attempt handshake and time out
+                    self.assertRaisesRegexp(ssl.SSLError, "timed out",
+                                            ssl.wrap_socket, c)
+                finally:
+                    c.close()
+                try:
+                    c = socket.socket(socket.AF_INET)
+                    c.settimeout(0.2)
+                    c = ssl.wrap_socket(c)
+                    # Will attempt handshake and time out
+                    self.assertRaisesRegexp(ssl.SSLError, "timed out",
+                                            c.connect, (host, port))
+                finally:
+                    c.close()
+            finally:
+                finish = True
+                t.join()
+                server.close()
+
 
 def test_main(verbose=False):
     if skip_expected: