Hi,
we had here some trouble with the proxy and installing hughe RPMs. In the end
we found the OOM killer working because the proxy load the complete RPM into
memory during delivery.
So I created two patches for the proxy:
0001: Is when sending content to the server via the proxy. Typically this way
not much data are transfered, but it can happen. So I removed the "body"
parameter from several functions and use "self.req" instead.
0002: This is when receiving data from the Server and deliver it to the
client. I used the SmartIO class to create a temp file when copying the data
from the response into the request. A direct usage of the FD was not possible.
It seems, that the response FD is already removed when the request function
try to read from it. So we need a temp file.
Note: SmartIO uses /tmp by default to create the tempfile. New systems
sometimes uses tmpfs on /tmp (Fedora?). So it might be a good idea to change
the default to a place without tmpfs.
--
Regards
Michael Calmer
--------------------------------------------------------------------------
Michael Calmer
SUSE LINUX Products GmbH, Maxfeldstr. 5, D-90409 Nuernberg
T: +49 (0) 911 74053 0
F: +49 (0) 911 74053575 - e-mail: michael.cal...@suse.com
--------------------------------------------------------------------------
SUSE LINUX Products GmbH, GF: Jeff Hawn, Jennifer Guild, Felix Imendörffer
HRB 16746 (AG Nürnberg)
>From 264b8a81d7d37b487de5849a048e8ad7869fd58c Mon Sep 17 00:00:00 2001
From: Michael Calmer <m...@suse.de>
Date: Thu, 18 Apr 2013 12:38:50 +0200
Subject: [PATCH 1/2] do not read data into memory which should be send to the
server
---
proxy/proxy/broker/rhnBroker.py | 3 +--
proxy/proxy/redirect/rhnRedirect.py | 4 ++--
proxy/proxy/rhnShared.py | 19 ++++++++++++-------
3 files changed, 15 insertions(+), 11 deletions(-)
diff --git a/proxy/proxy/broker/rhnBroker.py b/proxy/proxy/broker/rhnBroker.py
index 00f920e..ddd2e34 100644
--- a/proxy/proxy/broker/rhnBroker.py
+++ b/proxy/proxy/broker/rhnBroker.py
@@ -201,7 +201,6 @@ class BrokerHandler(SharedHandler):
% repr(_oto['X-RHN-Proxy-Auth']))
log_debug(3, 'Trying to connect to parent')
- data = self.req.read()
# Loops twice? Here's why:
# o If no errors, the loop is broken and we move on.
@@ -214,7 +213,7 @@ class BrokerHandler(SharedHandler):
# Add the proxy version
rhnFlags.get('outputTransportOptions')['X-RHN-Proxy-Version'] = str(_PROXY_VERSION)
- status = self._serverCommo(data) # part 2
+ status = self._serverCommo() # part 2
# check for proxy authentication blowup.
respHeaders = self.responseContext.getHeaders()
diff --git a/proxy/proxy/redirect/rhnRedirect.py b/proxy/proxy/redirect/rhnRedirect.py
index 7f73fd8..e633132 100644
--- a/proxy/proxy/redirect/rhnRedirect.py
+++ b/proxy/proxy/redirect/rhnRedirect.py
@@ -90,7 +90,7 @@ class RedirectHandler(SharedHandler):
self._connectToParent() # part 1
log_debug(4, 'Initiating communication with server...')
- status = self._serverCommo(self.req.read()) # part 2
+ status = self._serverCommo() # part 2
if (status != apache.OK) and (status != apache.HTTP_PARTIAL_CONTENT):
log_debug(3, "Leaving handler with status code %s" % status)
return status
@@ -413,7 +413,7 @@ class RedirectHandler(SharedHandler):
# redirected, so we can safely pass an empty string in as the request
# body.
- status = self._serverCommo('')
+ status = self._serverCommo()
# This little hack isn't pretty, but lets us normalize our result code.
diff --git a/proxy/proxy/rhnShared.py b/proxy/proxy/rhnShared.py
index c44fad8..ba6df9d 100644
--- a/proxy/proxy/rhnShared.py
+++ b/proxy/proxy/rhnShared.py
@@ -181,11 +181,10 @@ class SharedHandler:
port = None
return scheme, host, port, path
- def _serverCommo(self, data):
+ def _serverCommo(self):
""" Handler part 2
Server (or next proxy) communication.
- data is the self.req.read()
"""
log_debug(1)
@@ -199,7 +198,7 @@ class SharedHandler:
# Send the headers, the body and expect a response
try:
- status, headers, bodyFd = self._proxy2server(data)
+ status, headers, bodyFd = self._proxy2server()
self.responseContext.setHeaders(headers)
self.responseContext.setBodyFd(bodyFd)
except IOError:
@@ -325,13 +324,19 @@ class SharedHandler:
self._forwardHTTPBody(bodyFd, self.req)
- def _proxy2server(self, body):
+ def _proxy2server(self):
hdrs = rhnFlags.get('outputTransportOptions')
- log_debug(3, hdrs, "Content length", len(body))
+ log_debug(3, hdrs)
+ size = -1
# Put the headers into the output connection object
http_connection = self.responseContext.getConnection()
for (k, vals) in hdrs.items():
+ if k.lower() in ['content_length', 'content-length']:
+ try:
+ size = int(vals)
+ except ValueError:
+ pass
if k.lower() in ['content_length', 'content_type']:
# mod_wsgi modifies incoming headers so we have to transform them back
k = k.replace('_','-')
@@ -357,8 +362,8 @@ class SharedHandler:
http_connection.endheaders()
# Send the body too if there is a body
- if body:
- http_connection.send(body)
+ if size != 0:
+ http_connection.send(self.req.headers_in['wsgi.input'])
# At this point everything is sent to the server
# We now wait for the response
--
1.8.2
>From dfe7282f9d23746ce76aca085fc68b0870a8876a Mon Sep 17 00:00:00 2001
From: Michael Calmer <m...@suse.de>
Date: Thu, 18 Apr 2013 16:39:39 +0200
Subject: [PATCH 2/2] Do not read response data into memory
---
proxy/proxy/rhnShared.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/proxy/proxy/rhnShared.py b/proxy/proxy/rhnShared.py
index ba6df9d..ed18308 100644
--- a/proxy/proxy/rhnShared.py
+++ b/proxy/proxy/rhnShared.py
@@ -24,6 +24,7 @@ from types import ListType, TupleType
## global imports
from rhn import connections
from rhn.SSL import TimeoutException
+from rhn.SmartIO import SmartIO
## common imports
from rhn.UserDictCase import UserDictCase
@@ -447,11 +448,17 @@ class SharedHandler:
# read content if there is some or the size is unknown
if (size > 0 or size == -1) and (toRequest.method != 'HEAD'):
+ tfile = SmartIO(max_mem_size=CFG.BUFFER_SIZE)
buf = fromResponse.read(CFG.BUFFER_SIZE)
while buf:
try:
- toRequest.write(buf)
+ tfile.write(buf)
buf = fromResponse.read(CFG.BUFFER_SIZE)
except IOError:
buf = 0
+ tfile.seek(0)
+ if 'wsgi.file_wrapper' in toRequest.headers_in:
+ toRequest.output = toRequest.headers_in['wsgi.file_wrapper'](tfile, CFG.BUFFER_SIZE)
+ else:
+ toRequest.output = iter(lambda: tfile.read(CFG.BUFFER_SIZE), '')
--
1.8.2
_______________________________________________
Spacewalk-devel mailing list
Spacewalk-devel@redhat.com
https://www.redhat.com/mailman/listinfo/spacewalk-devel
