Hi all,
I work on GeoNode (http://geonode.org/). We're using OWSLib to fetch data
from GeoNetwork. Recently I've discovered that due to the way GeoNetwork
handles sessions, it makes a big difference for us to include HTTP cookies
on our service requests (without them, GeoNetwork creates a new session for
each service request and eventually locks up after running out of RAM.)
Therefore I would like to modify owslib to use a (persistent) instance of
urllib2.HTTPCookieProcessor when opening URLs, one which I can get a
reference to so that I can log in to GeoNetwork and have owslib use the
session cookie. Ideally, there would be a way to provide my own instance of
urllib2.OpenerDirector which I can customize as needed.
I am currently using a monkey patch to fix this behavior, but it's not the
way I'd like to do things (you can see the diff in GeoNode here:
https://github.com/dwins/geonode/compare/master...csw-cookies ).
For a deeper fix I see basically 3 options:
1) Use Python's urllib2.install_opener to set a default opener instance to
use, and be careful not to pass arguments into owslib methods that would
cause them to create their own openers (for example, including a username
and password in a call to owslib.util.openURL).
2) Create a module-level variable (owslib.util.opener) and modify
owslib.util.openURL and owslib.util.http_post to reference it exclusively.
Then client code which needs to modify the HTTP handling behavior can
simply replace this variable.
3) Make each client class (ie, owslib.csw.CatalogueServiceWeb) use its own
opener instance which may be provided by client code.
The attached patch is a partial implementation of approach #3. (In each of
these cases there is the question of how to handle the username/password
arguments to owslib.util.openURL, I haven't addressed this.)
Is this sort of thing something the OWSLib community would be interested in
incorporating? If so I'm happy to update this patch to incorporate feedback
etc.
--
David Winslow
OpenGeo - http://opengeo.org/
diff --git a/owslib/coverage/wcs100.py b/owslib/coverage/wcs100.py
index a0fd70b..5786d7e 100644
--- a/owslib/coverage/wcs100.py
+++ b/owslib/coverage/wcs100.py
@@ -11,7 +11,7 @@
from wcsBase import WCSBase, WCSCapabilitiesReader
from urllib import urlencode
-from owslib.util import openURL, testXMLValue
+from owslib.util import testXMLValue
from owslib.etree import etree
import os, errno
@@ -137,7 +137,7 @@ class WebCoverageService_1_0_0(WCSBase):
self.log.debug('WCS 1.0.0 DEBUG: Second part of URL: %s'%data)
- u=openURL(base_url, data, method, self.cookies)
+ u=self.openURL(base_url, data, method, self.cookies)
return u
diff --git a/owslib/coverage/wcs110.py b/owslib/coverage/wcs110.py
index d69ed6c..25edcc7 100644
--- a/owslib/coverage/wcs110.py
+++ b/owslib/coverage/wcs110.py
@@ -12,7 +12,7 @@
##########NOTE: Does not conform to new interfaces yet #################
from wcsBase import WCSBase, WCSCapabilitiesReader
-from owslib.util import openURL, testXMLValue
+from owslib.util import testXMLValue
from urllib import urlencode
from urllib2 import urlopen
from owslib.etree import etree
@@ -169,7 +169,7 @@ class WebCoverageService_1_1_0(WCSBase):
#encode and request
data = urlencode(request)
- u=openURL(base_url, data, method, self.cookies)
+ u=self.openURL(base_url, data, method, self.cookies)
return u
diff --git a/owslib/coverage/wcsBase.py b/owslib/coverage/wcsBase.py
index 147d1ab..2e82256 100644
--- a/owslib/coverage/wcsBase.py
+++ b/owslib/coverage/wcsBase.py
@@ -12,6 +12,7 @@
from urllib import urlencode
from urllib2 import urlopen, Request
from owslib.etree import etree
+from owslib.util import Client
import cgi
from StringIO import StringIO
@@ -19,7 +20,7 @@ from StringIO import StringIO
import logging
-class WCSBase(object):
+class WCSBase(Client):
"""Base class to be subclassed by version dependent WCS classes. Provides 'high-level' version independent methods"""
def __new__(self,url, xml, cookies):
""" overridden __new__ method
@@ -39,8 +40,8 @@ class WCSBase(object):
self._describeCoverage = {} #cache for DescribeCoverage responses
return obj
- def __init__(self):
- pass
+ def __init__(self, client=None):
+ super(WCSBase, self).__init__(client)
def getDescribeCoverage(self, identifier):
''' returns a describe coverage document - checks the internal cache to see if it has been fetched before '''
@@ -63,15 +64,16 @@ class WCSBase(object):
elif level=='CRITICAL':
self.log.setLevel(logging.CRITICAL)
-class WCSCapabilitiesReader(object):
+class WCSCapabilitiesReader(Client):
"""Read and parses WCS capabilities document into a lxml.etree infoset
"""
- def __init__(self, version=None, cookies = None):
+ def __init__(self, version=None, cookies = None, opener=None):
"""Initialize
@type version: string
@param version: WCS Version parameter e.g '1.0.0'
"""
+ super(WCSCapabilitiesReader, self).__init__(opener)
self.version = version
self._infoset = None
self.cookies = cookies
diff --git a/owslib/csw.py b/owslib/csw.py
index 213f049..02965ca 100644
--- a/owslib/csw.py
+++ b/owslib/csw.py
@@ -15,6 +15,7 @@ import random
from owslib.etree import etree
from owslib.filter import *
from owslib import util
+from owslib.util import Client
from owslib.ows import *
from owslib.iso import *
from owslib.fgdc import *
@@ -45,9 +46,9 @@ namespaces = {
schema_location = '%s %s' % (namespaces['csw'], schema)
-class CatalogueServiceWeb:
+class CatalogueServiceWeb(Client):
""" csw request class """
- def __init__(self, url, lang='en-US', version='2.0.2', timeout=10):
+ def __init__(self, url, lang='en-US', version='2.0.2', timeout=10, opener=None):
"""
Construct and process a GetCapabilities request
@@ -61,6 +62,7 @@ class CatalogueServiceWeb:
- timeout: timeout in seconds
"""
+ super(CatalogueServiceWeb, self).__init__(opener)
self.url = url
self.lang = lang
@@ -454,7 +456,7 @@ class CatalogueServiceWeb:
def _invoke(self):
# do HTTP request
- self.response = util.http_post(self.url, self.request, self.lang, self.timeout)
+ self.response = self.http_post(self.url, self.request, self.lang, self.timeout)
# parse result see if it's XML
self._exml = etree.parse(StringIO.StringIO(self.response))
diff --git a/owslib/util.py b/owslib/util.py
index 9419826..46e405f 100644
--- a/owslib/util.py
+++ b/owslib/util.py
@@ -34,56 +34,110 @@ class ServiceException(Exception):
#TODO: this should go in ows common module when refactored.
pass
-def openURL(url_base, data, method='Get', cookies=None, username=None, password=None):
- ''' function to open urls - wrapper around urllib2.urlopen but with additional checks for OGC service exceptions and url formatting, also handles cookies and simple user password authentication'''
- url_base.strip()
- lastchar = url_base[-1]
- if lastchar not in ['?', '&']:
- if url_base.find('?') == -1:
- url_base = url_base + '?'
+class Client(object):
+ def __init__(self, opener=None):
+ if opener is None:
+ self.opener = urllib2.build_opener()
else:
- url_base = url_base + '&'
-
- if username and password:
- # Provide login information in order to use the WMS server
- # Create an OpenerDirector with support for Basic HTTP
- # Authentication...
- passman = HTTPPasswordMgrWithDefaultRealm()
- passman.add_password(None, url_base, username, password)
- auth_handler = HTTPBasicAuthHandler(passman)
- opener = urllib2.build_opener(auth_handler)
- openit = opener.open
- else:
- openit = urlopen
-
- try:
- if method == 'Post':
- req = Request(url_base, data)
- else:
- req=Request(url_base + data)
- if cookies is not None:
- req.add_header('Cookie', cookies)
- u = openit(req)
- except HTTPError, e: #Some servers may set the http header to 400 if returning an OGC service exception or 401 if unauthorised.
- if e.code in [400, 401]:
- raise ServiceException, e.read()
- else:
- raise e
- # check for service exceptions without the http header set
- if u.info()['Content-Type'] in ['text/xml', 'application/xml']:
- #just in case 400 headers were not set, going to have to read the xml to see if it's an exception report.
- #wrap the url stram in a extended StringIO object so it's re-readable
- u=RereadableURL(u)
- se_xml= u.read()
- se_tree = etree.fromstring(se_xml)
- serviceException=se_tree.find('{http://www.opengis.net/ows}Exception')
- if serviceException is None:
- serviceException=se_tree.find('ServiceException')
- if serviceException is not None:
- raise ServiceException, \
- str(serviceException.text).strip()
- u.seek(0) #return cursor to start of u
- return u
+ self.opener = opener
+
+ def openURL(self, url_base, data, method='Get', cookies=None, username=None, password=None):
+ ''' function to open urls - wrapper around urllib2.urlopen but with additional checks for OGC service exceptions and url formatting, also handles cookies and simple user password authentication'''
+ url_base.strip()
+ lastchar = url_base[-1]
+ if lastchar not in ['?', '&']:
+ if url_base.find('?') == -1:
+ url_base = url_base + '?'
+ else:
+ url_base = url_base + '&'
+
+ if username and password:
+ # Provide login information in order to use the WMS server
+ # Create an OpenerDirector with support for Basic HTTP
+ # Authentication...
+ passman = HTTPPasswordMgrWithDefaultRealm()
+ passman.add_password(None, url_base, username, password)
+ auth_handler = HTTPBasicAuthHandler(passman)
+ self.opener.add_handler(auth_handler)
+
+ try:
+ if method == 'Post':
+ req = Request(url_base, data)
+ else:
+ req=Request(url_base + data)
+ if cookies is not None:
+ req.add_header('Cookie', cookies)
+ u = self.opener.open(req)
+ except HTTPError, e: #Some servers may set the http header to 400 if returning an OGC service exception or 401 if unauthorised.
+ if e.code in [400, 401]:
+ raise ServiceException, e.read()
+ else:
+ raise e
+ # check for service exceptions without the http header set
+ if u.info()['Content-Type'] in ['text/xml', 'application/xml']:
+ #just in case 400 headers were not set, going to have to read the xml to see if it's an exception report.
+ #wrap the url stram in a extended StringIO object so it's re-readable
+ u=RereadableURL(u)
+ se_xml= u.read()
+ se_tree = etree.fromstring(se_xml)
+ serviceException=se_tree.find('{http://www.opengis.net/ows}Exception')
+ if serviceException is None:
+ serviceException=se_tree.find('ServiceException')
+ if serviceException is not None:
+ raise ServiceException, \
+ str(serviceException.text).strip()
+ u.seek(0) #return cursor to start of u
+ return u
+
+
+ def http_post(self, url=None, request=None, lang='en-US', timeout=10):
+ """
+
+ Invoke an HTTP POST request
+
+ Parameters
+ ----------
+
+ - url: the URL of the server
+ - request: the request message
+ - lang: the language
+ - timeout: timeout in seconds
+
+ """
+ assert hasattr(self, 'opener')
+
+ if url is not None:
+ u = urlparse.urlsplit(url)
+ r = urllib2.Request(url, request)
+ r.add_header('User-Agent', 'OWSLib (http://trac.gispython.org/lab/wiki/OwsLib)')
+ r.add_header('Content-type', 'text/xml')
+ r.add_header('Content-length', '%d' % len(request))
+ r.add_header('Accept', 'text/xml')
+ r.add_header('Accept-Language', lang)
+ r.add_header('Accept-Encoding', 'gzip,deflate')
+ r.add_header('Host', u.netloc)
+
+ try:
+ up = self.opener.open(r, timeout=timeout);
+ except TypeError:
+ import socket
+ socket.setdefaulttimeout(timeout)
+ up = self.opener.open(r)
+
+ ui = up.info() # headers
+ response = up.read()
+ up.close()
+
+ # check if response is gzip compressed
+ if ui.has_key('Content-Encoding'):
+ if ui['Content-Encoding'] == 'gzip': # uncompress response
+ import gzip
+ cds = StringIO(response)
+ gz = gzip.GzipFile(fileobj=cds)
+ response = gz.read()
+
+ return response
+
#default namespace for nspath is OWS common
OWS_NAMESPACE = 'http://www.opengis.net/ows/1.1'
@@ -132,53 +186,6 @@ def testXMLValue(val, attrib=False):
return None
-def http_post(url=None, request=None, lang='en-US', timeout=10):
- """
-
- Invoke an HTTP POST request
-
- Parameters
- ----------
-
- - url: the URL of the server
- - request: the request message
- - lang: the language
- - timeout: timeout in seconds
-
- """
-
- if url is not None:
- u = urlparse.urlsplit(url)
- r = urllib2.Request(url, request)
- r.add_header('User-Agent', 'OWSLib (http://trac.gispython.org/lab/wiki/OwsLib)')
- r.add_header('Content-type', 'text/xml')
- r.add_header('Content-length', '%d' % len(request))
- r.add_header('Accept', 'text/xml')
- r.add_header('Accept-Language', lang)
- r.add_header('Accept-Encoding', 'gzip,deflate')
- r.add_header('Host', u.netloc)
-
- try:
- up = urllib2.urlopen(r,timeout=timeout);
- except TypeError:
- import socket
- socket.setdefaulttimeout(timeout)
- up = urllib2.urlopen(r)
-
- ui = up.info() # headers
- response = up.read()
- up.close()
-
- # check if response is gzip compressed
- if ui.has_key('Content-Encoding'):
- if ui['Content-Encoding'] == 'gzip': # uncompress response
- import gzip
- cds = StringIO(response)
- gz = gzip.GzipFile(fileobj=cds)
- response = gz.read()
-
- return response
-
def xml2string(xml):
"""
diff --git a/owslib/wms.py b/owslib/wms.py
index 3d5b0ab..f50ef97 100644
--- a/owslib/wms.py
+++ b/owslib/wms.py
@@ -18,7 +18,7 @@ Currently supports only version 1.1.1 of the WMS protocol.
import cgi
from urllib import urlencode
from etree import etree
-from .util import openURL
+from .util import Client
class ServiceException(Exception):
@@ -41,7 +41,7 @@ class CapabilitiesError(Exception):
pass
-class WebMapService(object):
+class WebMapService(Client):
"""Abstraction for OGC Web Map Service (WMS).
Implements IWebMapService.
@@ -56,9 +56,10 @@ class WebMapService(object):
def __init__(self, url, version='1.1.1', xml=None,
- username=None, password=None
+ username=None, password=None, opener=None
):
"""Initialize."""
+ super(WebMapService, self).__init__(opener);
self.url = url
self.username = username
self.password = password
@@ -220,7 +221,7 @@ class WebMapService(object):
data = urlencode(request)
- u = openURL(base_url, data, method, username = self.username, password = self.password)
+ u = self.openURL(base_url, data, method, username = self.username, password = self.password)
# check for service exceptions, and return
if u.info()['Content-Type'] == 'application/vnd.ogc.se_xml':
@@ -492,12 +493,13 @@ class ContactMetadata:
else: self.position = None
-class WMSCapabilitiesReader:
+class WMSCapabilitiesReader(Client):
"""Read and parse capabilities document into a lxml.etree infoset
"""
- def __init__(self, version='1.1.1', url=None, un=None, pw=None):
+ def __init__(self, version='1.1.1', url=None, un=None, pw=None, client=None):
"""Initialize"""
+ super(WMSCapabilitiesReader, self).__init__(client)
self.version = version
self._infoset = None
self.url = url
@@ -544,7 +546,7 @@ class WMSCapabilitiesReader:
#now split it up again to use the generic openURL function...
spliturl=getcaprequest.split('?')
- u = openURL(spliturl[0], spliturl[1], method='Get', username = self.username, password = self.password)
+ u = self.openURL(spliturl[0], spliturl[1], method='Get', username = self.username, password = self.password)
return etree.fromstring(u.read())
def readString(self, st):
_______________________________________________
Community mailing list
[email protected]
http://lists.gispython.org/mailman/listinfo/community
