http://bugs.python.org/issue6267
Cumulative patch to http and xmlrpc
diff --git a/Lib/BaseHTTPServer.py b/Lib/BaseHTTPServer.py
index acd8394..9bb3ba4 100644
--- a/Lib/BaseHTTPServer.py
+++ b/Lib/BaseHTTPServer.py
@@ -309,18 +309,26 @@
commands such as GET and POST.
"""
- self.raw_requestline = self.rfile.readline()
- if not self.raw_requestline:
+ try:
+ self.raw_requestline = self.rfile.readline()
+ if not self.raw_requestline:
+ self.close_connection = 1
+ return
+ if not self.parse_request():
+ # An error code has been sent, just exit
+ return
+ mname = 'do_' + self.command
+ if not hasattr(self, mname):
+ self.send_error(501, "Unsupported method (%r)" % self.command)
+ return
+ method = getattr(self, mname)
+ method()
+ self.wfile.flush() #actually send the response if not already done.
+ except socket.timeout, e:
+ #a read or a write timed out. Discard this connection
+ self.log_error("Request timed out: %r", e)
self.close_connection = 1
return
- if not self.parse_request(): # An error code has been sent, just exit
- return
- mname = 'do_' + self.command
- if not hasattr(self, mname):
- self.send_error(501, "Unsupported method (%r)" % self.command)
- return
- method = getattr(self, mname)
- method()
def handle(self):
"""Handle multiple requests if necessary."""
diff --git a/Lib/DocXMLRPCServer.py b/Lib/DocXMLRPCServer.py
index 432b3c3..4064ec2 100644
--- a/Lib/DocXMLRPCServer.py
+++ b/Lib/DocXMLRPCServer.py
@@ -240,10 +240,6 @@
self.end_headers()
self.wfile.write(response)
- # shut down the connection
- self.wfile.flush()
- self.connection.shutdown(1)
-
class DocXMLRPCServer( SimpleXMLRPCServer,
XMLRPCDocGenerator):
"""XML-RPC and HTML documentation server.
diff --git a/Lib/SimpleXMLRPCServer.py b/Lib/SimpleXMLRPCServer.py
index 967ac7e..5b5aced 100644
--- a/Lib/SimpleXMLRPCServer.py
+++ b/Lib/SimpleXMLRPCServer.py
@@ -106,6 +106,7 @@
import sys
import os
import traceback
+import re
try:
import fcntl
except ImportError:
@@ -430,6 +431,31 @@
# paths not on this list will result in a 404 error.
rpc_paths = ('/', '/RPC2')
+ #if not None, encode responses larger than this, if possible
+ encode_threshold = 1400 #a common MTU
+
+ #Override form StreamRequestHandler: full buffering of output
+ #and no Nagle.
+ wbufsize = -1
+ disable_nagle_algorithm = True
+
+ # a re to match a gzip Accept-Encoding
+ aepattern = re.compile(r"""
+ \s* ([^\s;]+) \s* #content-coding
+ (;\s* q \s*=\s* ([0-9\.]+))? #q
+ """, re.VERBOSE | re.IGNORECASE)
+
+ def accept_encodings(self):
+ r = {}
+ ae = self.headers.get("Accept-Encoding", "")
+ for e in ae.split(","):
+ match = self.aepattern.match(e)
+ if match:
+ v = match.group(3)
+ v = float(v) if v else 1.0
+ r[match.group(1)] = v
+ return r
+
def is_rpc_path_valid(self):
if self.rpc_paths:
return self.path in self.rpc_paths
@@ -463,6 +489,10 @@
size_remaining -= len(L[-1])
data = ''.join(L)
+ data = self.decode_request_content(data)
+ if data is None:
+ return #response has been sent
+
# In previous versions of SimpleXMLRPCServer, _dispatch
# could be overridden in this class, instead of in
# SimpleXMLRPCDispatcher. To maintain backwards compatibility,
@@ -481,18 +511,36 @@
self.send_header("X-exception", str(e))
self.send_header("X-traceback", traceback.format_exc())
+ self.send_header("Content-length", "0")
self.end_headers()
else:
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
+ if self.encode_threshold is not None:
+ if len(response) > self.encode_threshold:
+ q = self.accept_encodings().get("gzip", 0)
+ if q:
+ response = xmlrpclib.gzip_encode(response)
+ self.send_header("Content-Encoding", "gzip")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
- # shut down the connection
- self.wfile.flush()
- self.connection.shutdown(1)
+ def decode_request_content(self, data):
+ #support gzip encoding of request
+ encoding = self.headers.get("content-encoding", "identity").lower()
+ if encoding == "identity":
+ return data
+ if encoding == "gzip":
+ try:
+ return xmlrpclib.gzip_decode(data)
+ except ValueError:
+ self.send_response(400, "error decoding gzip content")
+ else:
+ self.send_response(501, "encoding %r not supported" % encoding)
+ self.send_header("Content-length", "0")
+ self.end_headers()
def report_404 (self):
# Report a 404 error
@@ -502,9 +550,6 @@
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
- # shut down the connection
- self.wfile.flush()
- self.connection.shutdown(1)
def log_request(self, code='-', size='-'):
"""Selectively log an accepted request."""
diff --git a/Lib/SocketServer.py b/Lib/SocketServer.py
index 7a25d0f..73cd219 100644
--- a/Lib/SocketServer.py
+++ b/Lib/SocketServer.py
@@ -445,6 +445,7 @@
def close_request(self, request):
"""Called to clean up an individual request."""
+ request.shutdown(socket.SHUT_WR)
request.close()
@@ -610,12 +611,11 @@
self.request = request
self.client_address = client_address
self.server = server
+ self.setup()
try:
- self.setup()
self.handle()
- self.finish()
finally:
- sys.exc_traceback = None # Help garbage collection
+ self.finish()
def setup(self):
pass
@@ -649,12 +649,17 @@
rbufsize = -1
wbufsize = 0
+ # A timeout to apply to the request socket, if not None.
+ timeout = None
+
# Disable nagle algoritm for this socket, if True.
# Use only when wbufsize != 0, to avoid small packets.
disable_nagle_algorithm = False
def setup(self):
self.connection = self.request
+ if self.timeout is not None:
+ self.connection.settimeout(self.timeout)
if self.disable_nagle_algorithm:
self.connection.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY, True)
diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py
index 8d527c3..ce561f4 100644
--- a/Lib/test/test_xmlrpc.py
+++ b/Lib/test/test_xmlrpc.py
@@ -273,7 +273,7 @@
# The evt is set twice. First when the server is ready to serve.
# Second when the server has been shutdown. The user must clear
# the event after it has been set the first time to catch the second set.
-def http_server(evt, numrequests):
+def http_server(evt, numrequests, requestHandler=None):
class TestInstanceClass:
def div(self, x, y):
return x // y
@@ -294,7 +294,9 @@
s.setblocking(True)
return s, port
- serv = MyXMLRPCServer(("localhost", 0),
+ if not requestHandler:
+ requestHandler = SimpleXMLRPCServer.SimpleXMLRPCRequestHandler
+ serv = MyXMLRPCServer(("localhost", 0), requestHandler,
logRequests=False, bind_and_activate=False)
try:
serv.socket.settimeout(3)
@@ -348,34 +350,36 @@
return False
-# NOTE: The tests in SimpleServerTestCase will ignore failures caused by
-# "temporarily unavailable" exceptions raised in SimpleXMLRPCServer. This
-# condition occurs infrequently on some platforms, frequently on others, and
-# is apparently caused by using SimpleXMLRPCServer with a non-blocking socket.
-# If the server class is updated at some point in the future to handle this
-# situation more gracefully, these tests should be modified appropriately.
-
-class SimpleServerTestCase(unittest.TestCase):
+class BaseServerTestCase(unittest.TestCase):
+ requestHandler = None
def setUp(self):
# enable traceback reporting
SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header = True
self.evt = threading.Event()
# start server thread to handle requests
- serv_args = (self.evt, 1)
+ serv_args = (self.evt, 1, self.requestHandler)
threading.Thread(target=http_server, args=serv_args).start()
# wait for the server to be ready
- self.evt.wait()
+ self.evt.wait(10)
self.evt.clear()
def tearDown(self):
# wait on the server thread to terminate
- self.evt.wait()
+ self.evt.wait(10)
# disable traceback reporting
SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header = False
+# NOTE: The tests in SimpleServerTestCase will ignore failures caused by
+# "temporarily unavailable" exceptions raised in SimpleXMLRPCServer. This
+# condition occurs infrequently on some platforms, frequently on others, and
+# is apparently caused by using SimpleXMLRPCServer with a non-blocking socket
+# If the server class is updated at some point in the future to handle this
+# situation more gracefully, these tests should be modified appropriately.
+
+class SimpleServerTestCase(BaseServerTestCase):
def test_simple1(self):
try:
p = xmlrpclib.ServerProxy(URL)
@@ -512,6 +516,110 @@
# This avoids waiting for the socket timeout.
self.test_simple1()
+#A test case that verifies that a server using the HTTP/1.1 keep-alive mechanism
+#does indeed serve subsequent requests on the same connection
+class KeepaliveServerTestCase(BaseServerTestCase):
+ #a request handler that supports keep-alive and logs requests into a
+ #class variable
+ class RequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
+ parentClass = SimpleXMLRPCServer.SimpleXMLRPCRequestHandler
+ protocol_version = 'HTTP/1.1'
+ myRequests = []
+ def handle(self):
+ self.myRequests.append([])
+ return self.parentClass.handle(self)
+ def handle_one_request(self):
+ result = self.parentClass.handle_one_request(self)
+ self.myRequests[-1].append(self.raw_requestline)
+ return result
+
+ requestHandler = RequestHandler
+ def setUp(self):
+ #clear request log
+ self.RequestHandler.myRequests = []
+ return BaseServerTestCase.setUp(self)
+
+ def test_two(self):
+ p = xmlrpclib.ServerProxy(URL)
+ self.assertEqual(p.pow(6,8), 6**8)
+ self.assertEqual(p.pow(6,8), 6**8)
+ self.assertEqual(len(self.RequestHandler.myRequests), 1)
+ #we may or may not catch the final "append" with the empty line
+ self.assertTrue(len(self.RequestHandler.myRequests[-1]) >= 2)
+
+#A test case that verifies that gzip encoding works in both directions
+#(for a request and the response)
+class GzipServerTestCase(BaseServerTestCase):
+ #a request handler that supports keep-alive and logs requests into a
+ #class variable
+ class RequestHandler(SimpleXMLRPCServer.SimpleXMLRPCRequestHandler):
+ parentClass = SimpleXMLRPCServer.SimpleXMLRPCRequestHandler
+ protocol_version = 'HTTP/1.1'
+
+ def do_POST(self):
+ #store content of last request in class
+ self.__class__.content_length = int(self.headers["content-length"])
+ return self.parentClass.do_POST(self)
+ requestHandler = RequestHandler
+
+ class Transport(xmlrpclib.Transport):
+ #custom transport, stores the response length for our perusal
+ fake_gzip = False
+ def parse_response(self, response):
+ self.response_length=int(response.getheader("content-length", 0))
+ return xmlrpclib.Transport.parse_response(self, response)
+
+ def send_content(self, connection, body):
+ if self.fake_gzip:
+ #add a lone gzip header to induce decode error remotely
+ connection.putheader("Content-Encoding", "gzip")
+ return xmlrpclib.Transport.send_content(self, connection, body)
+
+ def test_gzip_request(self):
+ t = self.Transport()
+ t.encode_threshold = None
+ p = xmlrpclib.ServerProxy(URL, transport=t)
+ self.assertEqual(p.pow(6,8), 6**8)
+ a = self.RequestHandler.content_length
+ t.encode_threshold = 0 #turn on request encoding
+ self.assertEqual(p.pow(6,8), 6**8)
+ b = self.RequestHandler.content_length
+ self.assertTrue(a>b)
+
+ def test_bad_gzip_request(self):
+ t = self.Transport()
+ t.encode_threshold = None
+ t.fake_gzip = True
+ p = xmlrpclib.ServerProxy(URL, transport=t)
+ cm = self.assertRaisesRegexp(xmlrpclib.ProtocolError,
+ re.compile(r"\b400\b"))
+ with cm:
+ p.pow(6, 8)
+
+ def test_gsip_response(self):
+ t = self.Transport()
+ p = xmlrpclib.ServerProxy(URL, transport=t)
+ old = self.requestHandler.encode_threshold
+ self.requestHandler.encode_threshold = None #no encoding
+ self.assertEqual(p.pow(6,8), 6**8)
+ a = t.response_length
+ self.requestHandler.encode_threshold = 0 #always encode
+ self.assertEqual(p.pow(6,8), 6**8)
+ b = t.response_length
+ self.requestHandler.encode_threshold = old
+ self.assertTrue(a>b)
+
+#Test special attributes of the ServerProxy object
+class ServerProxyTestCase(unittest.TestCase):
+ def test_close(self):
+ p = xmlrpclib.ServerProxy(URL)
+ self.assertEqual(p('close')(), None)
+
+ def test_transport(self):
+ t = xmlrpclib.Transport()
+ p = xmlrpclib.ServerProxy(URL, transport=t)
+ self.assertEqual(p('transport'), t)
+
# This is a contrived way to make a failure occur on the server side
# in order to test the _send_traceback_header flag on the server
class FailingMessageClass(mimetools.Message):
@@ -693,6 +801,9 @@
def makefile(self, x='r', y=-1):
raise RuntimeError
+ def close(self):
+ pass
+
class FakeTransport(xmlrpclib.Transport):
"""A Transport instance that records instead of sending a request.
@@ -703,7 +814,7 @@
def make_connection(self, host):
conn = xmlrpclib.Transport.make_connection(self, host)
- conn._conn.sock = self.fake_socket = FakeSocket()
+ conn.sock = self.fake_socket = FakeSocket()
return conn
class TransportSubclassTestCase(unittest.TestCase):
@@ -763,6 +874,9 @@
xmlrpc_tests = [XMLRPCTestCase, HelperTestCase, DateTimeTestCase,
BinaryTestCase, FaultTestCase, TransportSubclassTestCase]
xmlrpc_tests.append(SimpleServerTestCase)
+ xmlrpc_tests.append(KeepaliveServerTestCase)
+ xmlrpc_tests.append(GzipServerTestCase)
+ xmlrpc_tests.append(ServerProxyTestCase)
xmlrpc_tests.append(FailingServerTestCase)
xmlrpc_tests.append(CGIHandlerTestCase)
diff --git a/Lib/xmlrpclib.py b/Lib/xmlrpclib.py
index b58030f..160b61c 100644
--- a/Lib/xmlrpclib.py
+++ b/Lib/xmlrpclib.py
@@ -139,6 +139,10 @@
import re, string, time, operator
from types import *
+import gzip
+import socket
+import errno
+import httplib
# --------------------------------------------------------------------
# Internal stuff
@@ -1129,6 +1133,72 @@
p.close()
return u.close(), u.getmethodname()
+##
+# Encode a string using the gzip content encoding such as specified by the
+# Content-Encoding: gzip
+# in the HTTP header, as described in RFC 1952
+#
+# @param data the unencoded data
+# @return the encoded data
+
+def gzip_encode(data):
+ """data -> gzip encoded data
+
+ Encode data using the gzip content encoding as described in RFC 1952
+ """
+ f = StringIO.StringIO()
+ gzf = gzip.GzipFile(mode="wb", fileobj=f, compresslevel=1)
+ gzf.write(data)
+ gzf.close()
+ encoded = f.getvalue()
+ f.close()
+ return encoded
+
+##
+# Decode a string using the gzip content encoding such as specified by the
+# Content-Encoding: gzip
+# in the HTTP header, as described in RFC 1952
+#
+# @param data The encoded data
+# @return the unencoded data
+# @raises ValueError if data is not correctly coded.
+
+def gzip_decode(data):
+ """gzip encoded data -> unencoded data
+
+ Decode data using the gzip content encoding as described in RFC 1952
+ """
+ f = StringIO.StringIO(data)
+ gzf = gzip.GzipFile(mode="rb", fileobj=f)
+ try:
+ decoded = gzf.read()
+ except IOError:
+ raise ValueError("invalid data")
+ f.close()
+ gzf.close()
+ return decoded
+
+##
+# Return a decoded file-like object for the gzip encoding
+# as described in RFC 1952.
+#
+# @param response A stream supporting a read() method
+# @return a file-like object that the decoded data can be read() from
+
+class GzipDecodedResponse(gzip.GzipFile):
+ """a file-like object to decode a response encoded with the gzip
+ method, as described in RFC 1952.
+ """
+ def __init__(self, response):
+ #response doesn't support tell() and read(), required by
+ #GzipFile
+ self.stringio = StringIO.StringIO(response.read())
+ gzip.GzipFile.__init__(self, mode="rb", fileobj=self.stringio)
+
+ def close(self):
+ gzip.GzipFile.close(self)
+ self.stringio.close()
+
# --------------------------------------------------------------------
# request dispatcher
@@ -1156,8 +1226,39 @@
# client identifier (may be overridden)
user_agent = "xmlrpclib.py/%s (by www.pythonware.com)" % __version__
+ #if true, we'll request gzip encoding
+ accept_gzip_encoding = True
+
+ # if positive, encode request using gzip if it exceeds this threshold
+ # note that many server will get confused, so only use it if you know
+ # that they can decode such a request
+ encode_threshold = None #None = don't encode
+
def __init__(self, use_datetime=0):
self._use_datetime = use_datetime
+ self._connection = (None, None)
+ self._extra_headers = []
+ ##
+ # Send a complete request, and parse the response.
+ # Retry request if a cached connection has disconnected.
+ #
+ # @param host Target host.
+ # @param handler Target PRC handler.
+ # @param request_body XML-RPC request body.
+ # @param verbose Debugging flag.
+ # @return Parsed response.
+
+ def request(self, host, handler, request_body, verbose=0):
+ #retry request once if cached connection has gone cold
+ for i in (0, 1):
+ try:
+ return self.single_request(host, handler, request_body, verbose)
+ except (socket.error, httplib.HTTPException), e:
+ retry = (errno.ECONNRESET,
+ errno.ECONNABORTED,
+ httplib.BadStatusLine) #close after we sent request
+ if i or e[0] not in retry:
+ raise
##
# Send a complete request, and parse the response.
@@ -1168,30 +1269,39 @@
# @param verbose Debugging flag.
# @return Parsed response.
- def request(self, host, handler, request_body, verbose=0):
+ def single_request(self, host, handler, request_body, verbose=0):
# issue XML-RPC request
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
- self.send_request(h, handler, request_body)
- self.send_host(h, host)
- self.send_user_agent(h)
- self.send_content(h, request_body)
+ try:
+ self.send_request(h, handler, request_body)
+ self.send_host(h, host)
+ self.send_user_agent(h)
+ self.send_content(h, request_body)
- errcode, errmsg, headers = h.getreply(buffering=True)
+ response = h.getresponse(buffering=True)
+ if response.status == 200:
+ self.verbose = verbose
+ return self.parse_response(response)
+ except Fault:
+ raise
+ except Exception:
+ # All unexpected errors leave connection in
+ # a strange state, so we clear it.
+ self.close()
+ raise
- if errcode != 200:
- raise ProtocolError(
- host + handler,
- errcode, errmsg,
- headers
- )
-
- self.verbose = verbose
-
- return self.parse_response(h.getfile())
+ #discard any response data and raise exception
+ if (response.getheader("content-length", 0)):
+ response.read()
+ raise ProtocolError(
+ host + handler,
+ response.status, response.reason,
+ response.msg,
+ )
##
# Create parser.
@@ -1240,10 +1350,25 @@
# @return A connection handle.
def make_connection(self, host):
+ #return an existing connection if possible. This allows
+ #HTTP/1.1 keep-alive.
+ if self._connection and host == self._connection[0]:
+ return self._connection[1]
+
# create a HTTP connection object from a host descriptor
- import httplib
- host, extra_headers, x509 = self.get_host_info(host)
- return httplib.HTTP(host)
+ chost, self._extra_headers, x509 = self.get_host_info(host)
+ #store the host argument along with the connection object
+ self._connection = host, httplib.HTTPConnection(chost)
+ return self._connection[1]
+
+ ##
+ # Clear any cached connection object.
+ # Used in the event of socket errors.
+ #
+ def close(self):
+ if self._connection[1]:
+ self._connection[1].close()
+ self._connection = (None, None)
##
# Send request header.
@@ -1253,17 +1378,24 @@
# @param request_body XML-RPC body.
def send_request(self, connection, handler, request_body):
- connection.putrequest("POST", handler)
+ if (self.accept_gzip_encoding):
+ connection.putrequest("POST", handler, skip_accept_encoding=True)
+ connection.putheader("Accept-Encoding", "gzip")
+ else:
+ connection.putrequest("POST", handler)
##
# Send host name.
#
# @param connection Connection handle.
# @param host Host name.
+ #
+ # Note: This function doesn't actually add the "Host"
+ # header anymore, it is done as part of the connection.putrequest() in
+ # send_request() above.
def send_host(self, connection, host):
- host, extra_headers, x509 = self.get_host_info(host)
- connection.putheader("Host", host)
+ extra_headers = self._extra_headers
if extra_headers:
if isinstance(extra_headers, DictType):
extra_headers = extra_headers.items()
@@ -1286,6 +1418,13 @@
def send_content(self, connection, request_body):
connection.putheader("Content-Type", "text/xml")
+
+ #optionally encode the request
+ if (self.encode_threshold is not None and
+ self.encode_threshold < len(request_body)):
+ connection.putheader("Content-Encoding", "gzip")
+ request_body = gzip_encode(request_body)
+
connection.putheader("Content-Length", str(len(request_body)))
connection.endheaders(request_body)
@@ -1295,20 +1434,25 @@
# @param file Stream.
# @return Response tuple and target method.
- def parse_response(self, file):
- # read response from input file/socket, and parse it
+ def parse_response(self, response):
+ # read response data from httpresponse, and parse it
+ if response.getheader("Content-Encoding", "") == "gzip":
+ stream = GzipDecodedResponse(response)
+ else:
+ stream = response
p, u = self.getparser()
while 1:
- response = file.read(1024)
- if not response:
+ data = stream.read(1024)
+ if not data:
break
if self.verbose:
- print "body:", repr(response)
- p.feed(response)
+ print "body:", repr(data)
+ p.feed(data)
- file.close()
+ if stream is not response:
+ stream.close()
p.close()
return u.close()
@@ -1322,18 +1466,20 @@
# FIXME: mostly untested
def make_connection(self, host):
+ if self._connection and host == self._connection[0]:
+ return self._connection[1]
# create a HTTPS connection object from a host descriptor
# host may be a string, or a (host, x509-dict) tuple
- import httplib
- host, extra_headers, x509 = self.get_host_info(host)
try:
- HTTPS = httplib.HTTPS
+ HTTPS = httplib.HTTPSConnection
except AttributeError:
raise NotImplementedError(
"your version of httplib doesn't support HTTPS"
)
else:
- return HTTPS(host, None, **(x509 or {}))
+ chost, self._extra_headers, x509 = self.get_host_info(host)
+ self._connection = host, HTTPSConnection(chost, None, **(x509 or {}))
+ return self._connection[1]
##
# Standard server proxy. This class establishes a virtual connection
@@ -1398,6 +1544,9 @@
self.__verbose = verbose
self.__allow_none = allow_none
+ def __close(self):
+ self.__transport.close()
+
def __request(self, methodname, params):
# call a method on the remote server
@@ -1431,6 +1580,16 @@
# note: to call a remote object with an non-standard name, use
# result getattr(server, "strange-python-name")(args)
+ def __call__(self, attr):
+ """A workaround to get special attributes on the ServerProxy
+ without interfering with the magic __getattr__
+ """
+ if attr == "close":
+ return self.__close
+ elif attr == "transport":
+ return self.__transport
+ raise AttributeError("Attribute %r not found" % (attr,))
+
# compatibility
Server = ServerProxy