diff --git a/NEWS b/NEWS index 65884c2..0ee38d1 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,10 @@ * Sphinxified documentation. (Lukasz Balcerzak) + * The WSGI server now transparently handles when a git client submits data + using Content-Encoding: gzip. + (David Blewett) + * Added an entry point (dulwich.web.make_paster_app) for Paste.Deploy to be able to determine how to create an instance of HTTPGitApplication. (David Blewett) diff --git a/dulwich/gzip.py b/dulwich/gzip.py new file mode 100644 index 0000000..b47bea2 --- /dev/null +++ b/dulwich/gzip.py @@ -0,0 +1,71 @@ +from cStringIO import StringIO + +class StringConsumer(object): + + def __init__(self): + self._data = StringIO() + + def feed(self, data): + self._data.write(data) + + def close(self): + # We don't want to close the underlying StringIO instance + return self._data + +# The below courtesy of Fredrik Lundh +# http://effbot.org/zone/consumer-gzip.htm +# http://effbot.org/zone/copyright.htm , which contains this: +# "Unless otherwise noted, source code can be be used freely. +# Examples, test scripts and other short code fragments can be +# considered as being in the public domain." +class GzipConsumer(object): + + def __init__(self, consumer=None): + if consumer is None: + consumer = StringConsumer() + self._consumer = consumer + self._decoder = None + self._data = '' + + def feed(self, data): + if self._decoder is None: + # check if we have a full gzip header + data = self._data + data + try: + i = 10 + flag = ord(data[3]) + if flag & 4: # extra + x = ord(data[i]) + 256*ord(data[i+1]) + i = i + 2 + x + if flag & 8: # filename + while ord(data[i]): + i = i + 1 + i = i + 1 + if flag & 16: # comment + while ord(data[i]): + i = i + 1 + i = i + 1 + if flag & 2: # crc + i = i + 2 + if len(data) < i: + raise IndexError('not enough data') + if data[:3] != '\x1f\x8b\x08': + raise IOError('invalid gzip data') + data = data[i:] + except IndexError: + self.__data = data + return # need more data + import zlib + self._data = '' + self._decoder = zlib.decompressobj(-zlib.MAX_WBITS) + data = self._decoder.decompress(data) + if data: + self._consumer.feed(data) + + def close(self): + if self._decoder: + data = self._decoder.flush() + if data: + self._consumer.feed(data) + return self._consumer.close() + diff --git a/dulwich/tests/test_web.py b/dulwich/tests/test_web.py index 8509140..fa00514 100644 --- a/dulwich/tests/test_web.py +++ b/dulwich/tests/test_web.py @@ -19,6 +19,7 @@ """Tests for the Git HTTP server.""" from cStringIO import StringIO +import gzip import re from dulwich.object_store import ( @@ -52,6 +53,7 @@ from dulwich.web import ( get_info_packs, handle_service_request, _LengthLimitedFile, + GunzipFilter, HTTPGitRequest, HTTPGitApplication, ) @@ -417,20 +419,61 @@ class HTTPGitApplicationTestCase(TestCase): def setUp(self): super(HTTPGitApplicationTestCase, self).setUp() self._app = HTTPGitApplication('backend') + self._environ = { + 'PATH_INFO': '/foo', + 'REQUEST_METHOD': 'GET', + } - def test_call(self): - def test_handler(req, backend, mat): - # tests interface used by all handlers - self.assertEquals(environ, req.environ) - self.assertEquals('backend', backend) - self.assertEquals('/foo', mat.group(0)) - return 'output' - - self._app.services = { - ('GET', re.compile('/foo$')): test_handler, + def _test_handler(self, req, backend, mat): + # tests interface used by all handlers + self.assertEquals(self._environ, req.environ) + self.assertEquals('backend', backend) + self.assertEquals('/foo', mat.group(0)) + return 'output' + + def _add_handler(self, app): + req = self._environ['REQUEST_METHOD'] + app.services = { + (req, re.compile('/foo$')): self._test_handler, } - environ = { - 'PATH_INFO': '/foo', - 'REQUEST_METHOD': 'GET', - } - self.assertEquals('output', self._app(environ, None)) + + def test_call(self): + self._add_handler(self._app) + self.assertEquals('output', self._app(self._environ, None)) + +class GunzipTestCase(HTTPGitApplicationTestCase): + """TestCase for testing the GunzipFilter, ensuring the wsgi.input + is correctly decompressed and headers are corrected. + """ + + def setUp(self): + super(GunzipTestCase, self).setUp() + self._app = GunzipFilter(self._app) + self._environ['HTTP_CONTENT_ENCODING'] = 'gzip' + self._environ['REQUEST_METHOD'] = 'POST' + + def _get_zstream(self, text): + zstream = StringIO() + zfile = gzip.GzipFile(fileobj=zstream, mode='w') + zfile.write(text) + zfile.close() + return zstream + + def test_call(self): + self._add_handler(self._app.app) + orig = self.__class__.__doc__ + zstream = self._get_zstream(orig) + zlength = zstream.tell() + zstream.seek(0) + self.assertLess(zlength, len(orig)) + self.assertEquals(self._environ['HTTP_CONTENT_ENCODING'], + 'gzip') + self._environ['CONTENT_LENGTH'] = zlength + self._environ['wsgi.input'] = zstream + app_output = self._app(self._environ, None) + buf = self._environ['wsgi.input'] + self.assertIsNot(buf, zstream) + buf.seek(0) + self.assertEquals(orig, buf.read()) + self.assertLess(zlength, int(self._environ['CONTENT_LENGTH'])) + self.assertNotIn('HTTP_CONTENT_ENCODING', self._environ) diff --git a/dulwich/web.py b/dulwich/web.py index b55434b..bba57e7 100644 --- a/dulwich/web.py +++ b/dulwich/web.py @@ -19,6 +19,7 @@ """HTTP server for dulwich that implements the git smart HTTP protocol.""" from cStringIO import StringIO +import gzip import os import re import sys @@ -30,6 +31,7 @@ except ImportError: from dulwich._compat import parse_qs from dulwich import log_utils from dulwich.errors import NotGitRepository +from dulwich.gzip import GzipConsumer from dulwich.protocol import ( ReceivableProtocol, ) @@ -244,9 +246,11 @@ def handle_service_request(req, backend, mat): # TODO: git may used HTTP/1.1 chunked encoding instead of specifying # content-length content_length = req.environ.get('CONTENT_LENGTH', '') - if content_length: + if content_length and \ + not issubclass(input.__class__, _LengthLimitedFile): input = _LengthLimitedFile(input, int(content_length)) proto = ReceivableProtocol(input.read, write) + handler = handler_cls(backend, [url_prefix(mat)], proto, stateless_rpc=True) handler.handle() @@ -363,6 +367,48 @@ class HTTPGitApplication(object): return req.not_found('Sorry, that method is not supported') return handler(req, self.backend, mat) + +class GunzipFilter(object): + """WSGI middleware that unzips gzip-encoded requests before + passing on to the underlying application. + """ + + def __init__(self, application): + self.app = application + + def __call__(self, environ, start_response): + if environ.get('HTTP_CONTENT_ENCODING', '') == 'gzip': + zlength = int(environ.get('CONTENT_LENGTH', '0')) + consumer = GzipConsumer() + consumer.feed(environ['wsgi.input'].read(zlength)) + buf = consumer.close() + environ.pop('HTTP_CONTENT_ENCODING') + + environ['CONTENT_LENGTH'] = str(buf.tell()) + buf.seek(0) + environ['wsgi.input'] = buf + + return self.app(environ, start_response) + + +class LengthLimitedInput(object): + """WSGI middleware that limits the input length of a request to that + specified in Content-Length. + """ + + def __init__(self, application): + self.app = application + + def __call__(self, environ, start_response): + content_length = environ.get('CONTENT_LENGTH', '') + if content_length: + input = environ['wsgi.input'] + environ['wsgi.input'] = _LengthLimitedFile(input, + int(content_length)) + + return self.app(environ, start_response) + + def make_paster_app(global_config, **local_conf): """Factory function for a Paster WSGI app append_git=True will make each served git repo have .git appended to its @@ -492,7 +538,8 @@ try: log_utils.default_logging_config() backend = DictBackend({'/': Repo(gitdir)}) app = HTTPGitApplication(backend) - server = make_server(listen_addr, port, app, + wrapped_app = GunzipFilter(LengthLimitedInput(app)) + server = make_server(listen_addr, port, wrapped_app, handler_class=HTTPGitRequestHandler) logger.info('Listening for HTTP connections on %s:%d', listen_addr, port)