http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/isoresponse.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/isoresponse.py b/src/main/python/libraries/edge/opensearch/isoresponse.py new file mode 100644 index 0000000..70254c1 --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/isoresponse.py @@ -0,0 +1,38 @@ +import logging + +from xml.dom.minidom import * +import xml.sax.saxutils +from jinja2 import Environment, Template + +from edge.opensearch.response import Response + +class IsoResponse(Response): + def __init__(self): + self.env = Environment() + self.env.trim_blocks = True + self.env.autoescape = True + self.variables = {} + + def setTemplate(self, template): + self.template = self.env.from_string(template) + + def addNamespace(self, name, uri): + self.namespaces[name] = uri + + def removeNamespace(self, name): + del self.namespaces[name] + + def generate(self, pretty=False): + logging.debug('IsoResponse.generate is called.') + + if pretty: + try : + isoStr = self.template.render(self.variables).encode('utf-8').replace('\n', '') + except Exception as e: + logging.debug("Problem generating ISO " + str(e)) + del self.variables['doc'] + isoStr = self.template.render(self.variables).encode('utf-8').replace('\n', '') + document = xml.dom.minidom.parseString(isoStr) + return document.toprettyxml() + else: + return self.template.render(self.variables).replace('\n', '')
http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/isoresponsebysolr.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/isoresponsebysolr.py b/src/main/python/libraries/edge/opensearch/isoresponsebysolr.py new file mode 100644 index 0000000..fd9090b --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/isoresponsebysolr.py @@ -0,0 +1,121 @@ +import json +import logging + +from edge.opensearch.isoresponse import IsoResponse +from datetime import date, datetime + +class IsoResponseBySolr(IsoResponse): + def __init__(self): + super(IsoResponseBySolr, self).__init__() + + def generate(self, solrDatasetResponse, solrGranuleResponse = None, pretty=False): + self._populate(solrDatasetResponse, solrGranuleResponse) + return super(IsoResponseBySolr, self).generate(pretty) + + def _populate(self, solrDatasetResponse, solrGranuleResponse = None): + if solrDatasetResponse is not None: + solrJson = json.loads(solrDatasetResponse) + + logging.debug('dataset count: '+str(len(solrJson['response']['docs']))) + + if len(solrJson['response']['docs']) == 1: + # ok now populate variables! + doc = solrJson['response']['docs'][0] + + #self.variables['Dataset_ShortName'] = doc['Dataset-ShortName'][0] + #self.variables['Dataset_ShortName'] = u'unko' + + self.variables['doc'] = doc + + # Format dates + try: + self.variables['DatasetCitation_ReleaseDate'] = date.fromtimestamp(float(doc['DatasetCitation-ReleaseDateLong'][0]) / 1000).strftime('%Y%m%d') + self.variables['DatasetCoverage_StartTime'] = self._convertTimeLongToISO(doc['DatasetCoverage-StartTimeLong'][0]) + self.variables['DatasetCoverage_StopTime'] = self._convertTimeLongToISO(doc['DatasetCoverage-StopTimeLong'][0]) + except: + pass + + try: + # Create list of unique dataset sensor + self.variables['UniqueDatasetSensor'] = {} + for i, x in enumerate(doc['DatasetSource-Sensor-ShortName']): + self.variables['UniqueDatasetSensor'][x] = i + self.variables['UniqueDatasetSensor'] = self.variables['UniqueDatasetSensor'].values() + + # Create list of unique dataset source + self.variables['UniqueDatasetSource'] = {} + for i, x in enumerate(doc['DatasetSource-Source-ShortName']): + self.variables['UniqueDatasetSource'][x] = i + self.variables['UniqueDatasetSource'] = self.variables['UniqueDatasetSource'].values() + + # Replace all none, None values with empty string + doc['DatasetParameter-VariableDetail'] = [self._filterString(variableDetail) for variableDetail in doc['DatasetParameter-VariableDetail']] + + # Current date + self.variables['DateStamp'] = datetime.utcnow().strftime('%Y%m%d') + + # Data format version + self.variables['DatasetPolicy_DataFormat_Version'] = self._getDataFormatVersion(doc['DatasetPolicy-DataFormat'][0]) + except Exception as e: + logging.debug("Problem generating ISO " + str(e)) + del self.variables['doc'] + + if solrGranuleResponse is not None: + solrGranuleJson = json.loads(solrGranuleResponse) + + logging.debug('granule count: '+str(len(solrGranuleJson['response']['docs']))) + + for doc in solrGranuleJson['response']['docs']: + self._populateItem(solrGranuleResponse, doc, None) + + doc['Granule-StartTimeLong'][0] = self._convertTimeLongToISO(doc['Granule-StartTimeLong'][0]) + doc['Granule-StopTimeLong'][0] = self._convertTimeLongToISO(doc['Granule-StopTimeLong'][0]) + doc['Granule-ArchiveTimeLong'][0] = self._convertTimeLongToISO(doc['Granule-ArchiveTimeLong'][0]) + doc['Granule-CreateTimeLong'][0] = self._convertTimeLongToISO(doc['Granule-CreateTimeLong'][0]) + + # Create dictionary for bounding box extent + ''' + if ('GranuleReal-Value' in doc and 'GranuleReal-DatasetElement-Element-ShortName' in doc): + self.variables['GranuleBoundingBox'] = dict(zip(doc['GranuleReal-DatasetElement-Element-ShortName'], doc['GranuleReal-Value'])) + ''' + if 'GranuleSpatial-NorthLat' in doc and 'GranuleSpatial-EastLon' in doc and 'GranuleSpatial-SouthLat' in doc and 'GranuleSpatial-WestLon' in doc: + self.variables['GranuleBoundingBox'] = dict([('southernmostLatitude', doc['GranuleSpatial-SouthLat'][0]), + ('northernmostLatitude', doc['GranuleSpatial-NorthLat'][0]), + ('westernmostLongitude', doc['GranuleSpatial-WestLon'][0]), + ('easternmostLongitude', doc['GranuleSpatial-EastLon'][0])]) + break + + self.variables['granules'] = solrGranuleJson['response']['docs'] + + def _populateChannel(self, solrResponse): + pass + + def _populateItem(self, solrResponse, doc, item): + pass + + def _convertTimeLongToISO(self, time): + isoTime = '' + try: + isoTime = datetime.utcfromtimestamp(float(time) / 1000).isoformat() + 'Z' + except ValueError: + pass + return isoTime + + def _filterString(self, str): + if str.lower() == 'none': + return '' + else: + return str + + def _getDataFormatVersion(self, dataFormat): + version = '' + if dataFormat == 'NETCDF': + version = 3 + elif dataFormat == 'HDF': + version = 4 + else: + try: + version = int(dataFormat[-1]) + except: + pass + return version http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/response.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/response.py b/src/main/python/libraries/edge/opensearch/response.py new file mode 100644 index 0000000..6b68921 --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/response.py @@ -0,0 +1,12 @@ +import logging + +from xml.dom.minidom import Document +import xml.sax.saxutils + +class Response(object): + def __init__(self): + self.searchBasePath = '/ws/search/' + self.metadataBasePath = '/ws/metadata/' + + def generate(self, pretty=False): + pass http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/responsebysolr.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/responsebysolr.py b/src/main/python/libraries/edge/opensearch/responsebysolr.py new file mode 100644 index 0000000..eb01661 --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/responsebysolr.py @@ -0,0 +1,67 @@ +import json + +from edge.opensearch.response import Response + +class ResponseBySolr(Response): + def __init__(self): + super(ResponseBySolr, self).__init__() + + def generate(self, solrResponse): + self._populate(solrResponse) + return super(ResponseBySolr, self).generate() + + def _populate(self, solrResponse): + #response.title = 'OCSI Dataset Search: '+searchText + #response.description = 'Search result for "'+searchText+'"' + #response.link = searchUrl + self._populateChannel(solrResponse) + + if solrResponse is None: + self.variables.append( + {'namespace': 'openSearch', 'name': 'totalResults', 'value': 1} + ) + self.variables.append( + {'namespace': 'openSearch', 'name': 'startIndex', 'value': 1} + ) + self.variables.append( + {'namespace': 'openSearch', 'name': 'itemsPerPage', 'value': 1} + ) + item = [ + {'name': 'title', 'value': 'Error'}, + {'name': 'description', 'value': 'error'} + ] + self.items.append(item) + else: + #logging.debug(solrResponse) + solrJson = json.loads(solrResponse) + + self.variables.append( + {'namespace': 'openSearch', 'name': 'totalResults', 'value': solrJson['response']['numFound']} + ) + self.variables.append( + {'namespace': 'openSearch', 'name': 'startIndex', 'value': solrJson['response']['start']} + ) + self.variables.append( + {'namespace': 'openSearch', 'name': 'itemsPerPage', 'value': solrJson['responseHeader']['params']['rows']} + ) + + for doc in solrJson['response']['docs']: + """ + item = [ + {'name': 'title', 'value': doc['Dataset-LongName'][0]}, + {'name': 'description', 'value': doc['Dataset-Description'][0]}, + {'name': 'link', 'value': self._configuration.get('portal', 'datasetUrl')+'/'+doc['Dataset-ShortName'][0]} + ] + """ + item = [] + for docKey in doc.keys(): + item.append({'namespace': 'podaac', 'name': docKey, 'value': doc[docKey]}) + + self._populateItem(solrResponse, doc, item) + self.items.append(item) + + def _populateChannel(self, solrResponse): + pass + + def _populateItem(self, solrResponse, doc, item): + pass http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/responsewriter.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/responsewriter.py b/src/main/python/libraries/edge/opensearch/responsewriter.py new file mode 100644 index 0000000..2277c65 --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/responsewriter.py @@ -0,0 +1,142 @@ +from types import * +import logging +import urllib + +import requestresponder +from edge.httputility import HttpUtility +import math + +class ResponseWriter(requestresponder.RequestResponder): + def __init__(self, configFilePath, requiredParams = None): + super(ResponseWriter, self).__init__(configFilePath) + if requiredParams is None: + requiredParams = [] + self.requiredParams = requiredParams + self.searchParameters = {} + self.pretty = True + self.variables = {} + + def get(self, requestHandler): + super(ResponseWriter, self).get(requestHandler) + #check required parameters + for paramList in self.requiredParams: + countParamNotFound = 0 + for param in paramList: + try: + requestHandler.get_argument(param) + except: + countParamNotFound += 1 + if countParamNotFound == len(paramList): + raise Exception("One of the following parameters is required: " + ', '.join(paramList)) + + def _constructSingleSolrDatasetQuery(self, variables): + queries = [] + for key, value in variables.iteritems(): + # Only key used for ISO granule record is dataset + if key == 'datasetId': + query = 'Dataset-PersistentId:'+self._urlEncodeSolrQueryValue(value) + queries.append(query) + elif key == 'shortName': + query = 'Dataset-ShortName-Full:'+self._urlEncodeSolrQueryValue(value) + queries.append(query) + + if len(queries) == 0: + queries.append('*') + + query = 'q='+'+AND+'.join(queries)+'&fq=DatasetPolicy-AccessType-Full:(OPEN+OR+PREVIEW+OR+SIMULATED+OR+REMOTE)+AND+DatasetPolicy-ViewOnline:Y&version=2.2&rows=1&indent=on&wt=json' + logging.debug('solr query: '+query) + + return query + + def _getSingleSolrDatasetResponse(self, variables, callback): + query = self._constructSingleSolrDatasetQuery(variables) + url = self._configuration.get('solr', 'datasetUrl') + + httpUtility = HttpUtility() + return httpUtility.getResponse(url+'/select/?'+query, callback) + + def _urlEncodeSolrQueryValue(self, value): + return urllib.quote('"'+value+'"') + + def _constructBoundingBoxQuery(self, value): + coords = value.split(",") + if len(coords) < 4: + return None + try: + west = float(coords[0]) + south = float(coords[1]) + east = float(coords[2]) + north = float(coords[3]) + + centerY = (south + north) / 2 + halfHeight = math.fabs(north - south) / 2 + + #Check if we need to split box into two + if (east < west): + west1 = west + east1 = 180.0 + + centerX1 = (west1 + east1) / 2 + halfWidth1 = math.fabs(east1 - west1) / 2 + + west2 = -180.0 + east2 = east + + centerX2 = (west2 + east2) / 2 + halfWidth2 = math.fabs(east2 - west2) / 2 + + return "fq={!frange+l=1+u=2}map(sum(" + self._solrSeparatingXAxisFunctionQueryAggregate(centerX1, halfWidth1, centerX2, halfWidth2) + "," + self._solrSeparatingYAxisFunctionQuery(centerY, halfHeight) + "),0,0,1,0)&fq=CenterY:*" + else: + centerX = (west + east) / 2 + halfWidth = math.fabs(east - west) / 2 + + return "fq={!frange+l=1+u=2}map(sum(" + self._solrSeparatingXAxisFunctionQuery(centerX, halfWidth) + "," + self._solrSeparatingYAxisFunctionQuery(centerY, halfHeight) + "),0,0,1,0)&fq=CenterY:*" + except: + return None + + def _solrSeparatingXAxisFunctionQuery(self, center, width): + return "map(sum(" + self._solrSeparatingAxixFunction(center, "CenterX1", width, "HalfWidth1", 360) + "," + self._solrSeparatingXAxixFunctionQueryPossibleNullWidth(center, width) + "),0,1,0,1)" + + def _solrSeparatingXAxisFunctionQueryAggregate(self, center1, width1, center2, width2): + return "map(sum(" + self._solrSeparatingAxixFunction(center1, "CenterX1", width1, "HalfWidth1", 360) + ","+ self._solrSeparatingXAxixFunctionQueryPossibleNullWidth(center1, width1) + "," +self._solrSeparatingAxixFunction(center2, "CenterX1", width2, "HalfWidth1", 360) + "," + self._solrSeparatingXAxixFunctionQueryPossibleNullWidth(center2, width2) +"),0,3,0,1)" + + def _solrSeparatingYAxisFunctionQuery(self, center, height): + return self._solrSeparatingAxixFunction(center, "CenterY", height, "HalfHeight", 180) + + def _solrSeparatingAxixFunction(self, center, centerVar, length, lengthVar, span): + return "map(sub(abs(sub(%.15g,%s)),sum(%.15g,%s)),0,%i,1,0)" % (center, centerVar, length, lengthVar, span) + + def _solrSeparatingXAxixFunctionQueryPossibleNullWidth(self, center, length): + return "map(sum(map(HalfWidth2,0,0,1,0),map(sub(abs(sub(%.15g,CenterX2)),sum(%.15g,HalfWidth2)),0,360,1,0)),0,0,0,1)" % (center, length) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + pass + + def _onSolrResponse(self, response): + if response.error: + self._handleException(str(response.error)) + else: + self._writeResponse(response.body) + + def _writeResponse(self, responseText): + searchText = '' + if 'keyword' in self.variables: + searchText = self.variables['keyword'] + try: + openSearchResponse = self._generateOpenSearchResponse( + responseText, + searchText, + self._configuration.get('service', 'url') + self.requestHandler.request.path, + self.searchParameters, + self.pretty + ) + self.requestHandler.set_header("Content-Type", "application/xml") + self.requestHandler.write(openSearchResponse) + self.requestHandler.finish() + except BaseException as exception: + self._handleException(str(exception)) + + def _handleException(self, error): + self.requestHandler.set_status(404) + self.requestHandler.write(error) + self.requestHandler.finish() http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/rssresponse.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/rssresponse.py b/src/main/python/libraries/edge/opensearch/rssresponse.py new file mode 100644 index 0000000..d36a109 --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/rssresponse.py @@ -0,0 +1,126 @@ +import logging + +from xml.dom.minidom import Document +import xml.sax.saxutils + +from edge.opensearch.response import Response + +class RssResponse(Response): + def __init__(self): + super(RssResponse, self).__init__() + self.namespaces = { + 'opensearch': 'http://a9.com/-/spec/opensearch/1.1/', + 'podaac': 'http://podaac.jpl.nasa.gov/opensearch/', + 'georss': 'http://www.georss.org/georss', + 'gml': 'http://www.opengis.net/gml', + 'time': 'http://a9.com/-/opensearch/extensions/time/1.0/', + 'atom': 'http://www.w3.org/2005/Atom' + } + + self.title = None + self.link = None + self.description = None + self.variables = [] + self.items = [] + self.parameters = {} + + def addNamespace(self, name, uri): + self.namespaces[name] = uri + + def removeNamespace(self, name): + del self.namespaces[name] + + def generate(self, pretty=False): + logging.debug('RssResponse.generate is called.') + + document = Document() + rss = document.createElement('rss') + rss.setAttribute('version', '2.0') + for namespace in self.namespaces.keys(): + rss.setAttribute('xmlns:'+namespace, self.namespaces[namespace]) + document.appendChild(rss) + + channel = document.createElement('channel') + rss.appendChild(channel) + + title = document.createElement('title') + channel.appendChild(title) + title.appendChild(document.createTextNode(xml.sax.saxutils.escape(self.title))) + + description = document.createElement('description') + channel.appendChild(description) + description.appendChild(document.createTextNode(xml.sax.saxutils.escape(self.description))) + + link = document.createElement('link') + channel.appendChild(link) + link.appendChild(document.createTextNode(xml.sax.saxutils.escape(self.link))) + + for variable in self.variables: + ''' + elementName = variable['name'] + if 'namespace' in variable: + elementName = variable['namespace']+':'+elementName + + variableElement = document.createElement(elementName) + channel.appendChild(variableElement) + variableElement.appendChild(document.createTextNode(xml.sax.saxutils.escape(str(variable['value'])))) + ''' + self._createNode(document, variable, channel) + + for item in self.items: + itemElement = document.createElement('item') + channel.appendChild(itemElement) + + for itemEntry in item: + self._createNode(document, itemEntry, itemElement); + ''' + elementName = itemEntry['name'] + if 'namespace' in itemEntry: + elementName = itemEntry['namespace']+':'+elementName + + variableElement = document.createElement(elementName) + itemElement.appendChild(variableElement) + + value = itemEntry['value'] + if isinstance(value, list): + if len(value) > 1: + for valueEntry in value: + valueName = 'value' + if 'namespace' in itemEntry: + valueName = itemEntry['namespace']+':'+valueName + valueElement = document.createElement(valueName) + variableElement.appendChild(valueElement) + valueElement.appendChild(document.createTextNode(xml.sax.saxutils.escape(str(valueEntry)))) + else: + variableElement.appendChild(document.createTextNode(xml.sax.saxutils.escape(str(value[0])))) + elif isinstance(value, dict): + for key in value.keys(): + valueName = key + if 'namespace' in itemEntry: + valueName = itemEntry['namespace']+':'+valueName + valueElement = document.createElement(valueName) + variableElement.appendChild(valueElement) + valueElement.appendChild(document.createTextNode(xml.sax.saxutils.escape(str(value[key])))) + else: + variableElement.appendChild(document.createTextNode(xml.sax.saxutils.escape(str(value)))) + ''' + return document.toprettyxml() if pretty else document.toxml('utf-8') + + def _createNode(self, document, itemEntry, itemElement): + elementName = itemEntry['name'] + if 'namespace' in itemEntry: + elementName = itemEntry['namespace']+':'+elementName + variableElement = document.createElement(elementName) + itemElement.appendChild(variableElement) + if 'value' in itemEntry: + value = itemEntry['value'] + if isinstance(value, list): + for valueEntry in value: + self._createNode(document, valueEntry, variableElement) + elif isinstance(value, dict): + self._createNode(document, value, variableElement) + else: + variableElement.appendChild(document.createTextNode(xml.sax.saxutils.escape(str(value)))) + if 'attribute' in itemEntry: + for attr in itemEntry['attribute'].keys(): + variableElement.setAttribute(attr, itemEntry['attribute'][attr]) http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/rssresponsebysolr.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/rssresponsebysolr.py b/src/main/python/libraries/edge/opensearch/rssresponsebysolr.py new file mode 100644 index 0000000..fffe234 --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/rssresponsebysolr.py @@ -0,0 +1,134 @@ +import json +import urllib + +from edge.opensearch.rssresponse import RssResponse +from collections import defaultdict + +class RssResponseBySolr(RssResponse): + def __init__(self): + super(RssResponseBySolr, self).__init__() + + def generate(self, solrResponse, pretty=False): + self._populate(solrResponse) + return super(RssResponseBySolr, self).generate(pretty) + + def _populate(self, solrResponse): + #response.title = 'OCSI Dataset Search: '+searchText + #response.description = 'Search result for "'+searchText+'"' + #response.link = searchUrl + self._populateChannel(solrResponse) + + if solrResponse is None: + self.variables.append( + {'namespace': 'opensearch', 'name': 'totalResults', 'value': 1} + ) + self.variables.append( + {'namespace': 'opensearch', 'name': 'startIndex', 'value': 1} + ) + self.variables.append( + {'namespace': 'opensearch', 'name': 'itemsPerPage', 'value': 1} + ) + self.parameters['startIndex'] = 0 + url = self.link + '?' + urllib.urlencode(self.parameters) + self.variables.append({'namespace': 'atom', 'name': 'link', 'attribute': {'href': url, 'rel': 'self', 'type': 'application/rss+xml'}}) + self.variables.append({'namespace': 'atom', 'name': 'link', 'attribute': {'href': url, 'rel': 'first', 'type': 'application/rss+xml'}}) + item = [ + {'name': 'title', 'value': 'Error'}, + {'name': 'description', 'value': 'error'} + ] + self.items.append(item) + else: + #logging.debug(solrResponse) + solrJson = json.loads(solrResponse) + numFound = int(solrJson['response']['numFound']) + start = int(solrJson['response']['start']) + rows = int(solrJson['responseHeader']['params']['rows']) + + self.parameters['startIndex'] = start + self.variables.append({'namespace': 'atom', 'name': 'link', 'attribute': {'href': self.link + '?' + urllib.urlencode(self.parameters), 'rel': 'self', 'type': 'application/rss+xml'}}) + self.parameters['startIndex'] = 0 + self.variables.append({'namespace': 'atom', 'name': 'link', 'attribute': {'href': self.link + '?' + urllib.urlencode(self.parameters), 'rel': 'first', 'type': 'application/rss+xml'}}) + if start > 0: + if (start - rows > 0): + self.parameters['startIndex'] = start - rows + self.variables.append({'namespace': 'atom', 'name': 'link', 'attribute': {'href': self.link + '?' + urllib.urlencode(self.parameters), 'rel': 'previous', 'type': 'application/rss+xml'}}) + if start + rows < numFound: + self.parameters['startIndex'] = start + rows + self.variables.append({'namespace': 'atom', 'name': 'link', 'attribute': {'href': self.link + '?' + urllib.urlencode(self.parameters), 'rel': 'next', 'type': 'application/rss+xml'}}) + + self.variables.append( + {'namespace': 'opensearch', 'name': 'totalResults', 'value': solrJson['response']['numFound']} + ) + self.variables.append( + {'namespace': 'opensearch', 'name': 'startIndex', 'value': solrJson['response']['start']} + ) + self.variables.append( + {'namespace': 'opensearch', 'name': 'itemsPerPage', 'value': solrJson['responseHeader']['params']['rows']} + ) + + for doc in solrJson['response']['docs']: + """ + item = [ + {'name': 'title', 'value': doc['Dataset-LongName'][0]}, + {'name': 'description', 'value': doc['Dataset-Description'][0]}, + {'name': 'link', 'value': self._configuration.get('portal', 'datasetUrl')+'/'+doc['Dataset-ShortName'][0]} + ] + """ + item = [] + ''' + #Handle dataset_location_policy values differently + if 'DatasetLocationPolicy-Type' in doc and 'DatasetLocationPolicy-BasePath' in doc: + for i, x in enumerate(doc['DatasetLocationPolicy-Type']): + item.append({'namespace': 'podaac', 'name': self._camelCaseStripHyphen(x.title()), 'value': doc['DatasetLocationPolicy-BasePath'][i]}) + del doc['DatasetLocationPolicy-Type'] + del doc['DatasetLocationPolicy-BasePath'] + + multiValuedElementsKeys = ('DatasetRegion-', 'DatasetCharacter-', 'DatasetCitation-', 'DatasetContact-Contact-', 'DatasetDatetime-', + 'DatasetInteger-', 'DatasetParameter-', 'DatasetProject-', 'DatasetReal-', 'DatasetResource-', + 'DatasetSoftware-', 'DatasetSource-', 'DatasetVersion-', 'Collection-', + 'GranuleArchive-', 'GranuleReference-', 'GranuleReal-') + multiValuedElements = defaultdict(list) + for docKey in doc.keys(): + if docKey.startswith(multiValuedElementsKeys): + multiValuedElements[docKey.split('-', 1)[0]].append(docKey) + else: + item.append({'namespace': 'podaac', 'name': self._camelCaseStripHyphen(docKey), 'value': doc[docKey]}) + for multiValuedKey in multiValuedElements: + for i, x in enumerate(doc[multiValuedElements[multiValuedKey][0]]): + values = {} + for key in multiValuedElements[multiValuedKey]: + values[self._camelCaseStripHyphen(key.split('-', 1)[1])] = doc[key][i] + item.append({'namespace': 'podaac', 'name': self._camelCaseStripHyphen(multiValuedKey), 'value': values}) + ''' + self._populateItem(solrResponse, doc, item) + self.items.append(item) + + def _populateChannel(self, solrResponse): + pass + + def _populateItem(self, solrResponse, doc, item): + pass + + def _populateItemWithPodaacMetadata(self, doc, item, multiValuedElementsKeys): + ignoreElementsEndingWith = ('-Full', '-Long') + multiValuedElements = defaultdict(list) + for docKey in doc.keys(): + if docKey.startswith(multiValuedElementsKeys): + multiValuedElements[docKey.split('-', 1)[0]].append(docKey) + elif not docKey.endswith(ignoreElementsEndingWith): + if len(doc[docKey]) > 1: + item.append({'namespace': 'podaac', 'name': self._camelCaseStripHyphen(docKey), 'value': [{'namespace': 'podaac', 'name': 'value', 'value': x} for x in doc[docKey]]}) + else: + item.append({'namespace': 'podaac', 'name': self._camelCaseStripHyphen(docKey), 'value': doc[docKey][0]}) + for multiValuedKey in multiValuedElements: + for i, x in enumerate(doc[multiValuedElements[multiValuedKey][0]]): + values = [] + for key in multiValuedElements[multiValuedKey]: + if not key.endswith(ignoreElementsEndingWith): + values.append({'namespace': 'podaac', 'name': self._camelCaseStripHyphen(key.split('-', 1)[1]), 'value': doc[key][i]}) + item.append({'namespace': 'podaac', 'name': self._camelCaseStripHyphen(multiValuedKey), 'value': values}) + + def _camelCaseStripHyphen(self, key): + #special case to remove duplicate element, contact from element tag + key = key.replace('-Element-', '', 1).replace('Contact-', '', 1) + return key[0].lower() + key[1:].replace('-', '') http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/solrcmrtemplateresponse.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/solrcmrtemplateresponse.py b/src/main/python/libraries/edge/opensearch/solrcmrtemplateresponse.py new file mode 100644 index 0000000..0dbcebb --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/solrcmrtemplateresponse.py @@ -0,0 +1,243 @@ +import datetime +import pycurl +from StringIO import StringIO +import json +import logging +import urllib +import os.path + +from edge.opensearch.templateresponse import TemplateResponse + +class SolrCmrTemplateResponse(TemplateResponse): + def __init__(self, configuration, link, parameters): + super(SolrCmrTemplateResponse, self).__init__() + self._configuration = configuration + self.link = link + self.parameters = parameters + + def generate(self, solrResponse, pretty=False): + self._populate(solrResponse) + return super(SolrCmrTemplateResponse, self).generate(pretty) + + def _get_cmr_response(self, url, cmr): + + # Execute the curl command and load the json object. + buffer = StringIO() + try: + c = pycurl.Curl() + c.setopt(c.URL, url) + c.setopt(c.WRITEDATA, buffer) + c.perform() + c.close() + except: + output = '{ "errors": "%s" }' %(c.errstr()) + c.close() + else: + try: + output = json.loads(buffer.getvalue()) + except: + output = '{ "errors": "json loads failed: [%s]" }' %(buffer.getvalue()) + + # Format the output if there are errors. + response = {} + if output.keys().__contains__('errors'): + response['entry'] = [] + response['updated'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + response['id'] = url + if url.find('collections') != -1: + response['title'] = 'ECHO dataset metadata' + elif url.find('granules') != -1: + response['title'] = 'ECHO granule metadata' + response['errors'] = output['errors'][0] + else: + response = output + + try: + + if response.keys().__contains__('errors'): + return(response) + + if not(response.keys().__contains__('feed')): + raise ValueError('no "feed" in the cmr response') + if not(response['feed'].keys().__contains__('entry')): + raise ValueError('no "entry" in the cmr response') + if not(response['feed'].keys().__contains__('updated')): + raise ValueError('no "updated" key in the cmr response') + if not(response['feed'].keys().__contains__('id')): + raise ValueError('no "id" key in the cmr response') + if not(response['feed'].keys().__contains__('title')): + raise ValueError('no "id" key in the cmr response') + + # Create lists if they do not exists. + if not(cmr.keys().__contains__('cmr_search_updated')): + cmr['cmr_search_updated'] = [] + if not(cmr.keys().__contains__('cmr_search_url')): + cmr['cmr_search_url'] = [] + if not(cmr.keys().__contains__('cmr_search_title')): + cmr['cmr_search_title'] = [] + + cmr['cmr_search_updated'].append(response['feed']['updated']) + cmr['cmr_search_url'].append(response['feed']['id']) + cmr['cmr_search_title'].append(response['feed']['title']) + + # Create list objects for the "entry" key. + if response['feed']['entry'] != []: + entry = response['feed']['entry'][0] + for key in entry: + keyname = 'cmr_%s' %(key) + if not(cmr.keys().__contains__(keyname)): + cmr[keyname] = [] + cmr[keyname].append(entry[key]) + + except ValueError, e: + msg = 'Error! parse error: %s.' %e + print '%s\n' %msg + + return(cmr) + + def _populate(self, solrResponse): + self.variables['link'] = self.link + + start = 0 + rows = 0 + numFound = 0 + + if solrResponse is not None: + solrJson = json.loads(solrResponse) + + logging.debug('Total doc count: '+str(solrJson['response']['numFound'])) + logging.debug('Total item count: '+ str(len(solrJson['response']['docs']))) + + #---------------------------------------------------------------------------------------------- + # CMR: Processing + #---------------------------------------------------------------------------------------------- + cmr_total_time = datetime.timedelta(0) + cmr_total_count = 0 + + for i in range(len(solrJson['response']['docs'])): + + doc = solrJson['response']['docs'][i] + logging.debug('doc[id]: '+str(doc['id'])) + + #------------------------------------------------------------------------------------------ + # CMR: Initialize + #------------------------------------------------------------------------------------------ + cmr = {} + + #------------------------------------------------------------------------------------------ + # CMR: PRODUCT_TYPE + #------------------------------------------------------------------------------------------ + + if solrJson['response']['docs'][i].keys().__contains__('product_type_dataset_short_name_list'): + + for j in range(len(doc['product_type_dataset_short_name_list'])): + + cmr_test = 0 + if cmr_test == 1: + cmr_search_url = 'https://cmr.earthdata.nasa.gov/search/collections.json?keyword=NSIDC' + else: + cmr_search_url = 'https://cmr.earthdata.nasa.gov/search/collections.json?keyword=' + \ + doc['product_type_dataset_short_name_list'][j] + + logging.debug('cmr search url: ' +cmr_search_url) + time1 = datetime.datetime.now() + cmr = self._get_cmr_response(cmr_search_url, cmr) + time2 = datetime.datetime.now() + time3 = time2 - time1 + logging.debug('cmr search time: ' + time3.__str__() + ' 1 product_type') + + cmr_total_time = cmr_total_time + time3 + cmr_total_count = cmr_total_count + 1 + + #------------------------------------------------------------------------------------------ + # CMR: PRODUCT (Only search when the query contains 'id' - ie individual cmr search) + #------------------------------------------------------------------------------------------ + + elif solrJson['response']['docs'][i].keys().__contains__('product_granule_remote_granule_ur_list') and \ + self.parameters.keys().__contains__('id'): + + for j in range(len(doc['product_granule_remote_granule_ur_list'])): + + cmr_test = 0 + if cmr_test == 1: + cmr_search_url = 'https://cmr.earthdata.nasa.gov/search/granules.json?granule_ur[]=' + \ + '20070106-NAR18_SST-EUR-L2P-sst1nar_noaa18_20070106_asc-v01.nc' + else: + cmr_search_url = 'https://cmr.earthdata.nasa.gov/search/granules.json?granule_ur[]=' + \ + doc['product_granule_remote_granule_ur_list'][j] + + logging.debug('cmr search url: ' +cmr_search_url) + time1 = datetime.datetime.now() + + cmr = self._get_cmr_response(cmr_search_url, cmr) + + time2 = datetime.datetime.now() + time3 = time2 - time1 + logging.debug('cmr search time: ' + time3.__str__() + ' 1 product') + + cmr_total_time = cmr_total_time + time3 + cmr_total_count = cmr_total_count + 1 + + #------------------------------------------------------------------------------------------ + # CMR: Docs not containing CMR search related criteria + #------------------------------------------------------------------------------------------ + else: + continue + + #------------------------------------------------------------------------------------------ + # CMR: Post-processing + #------------------------------------------------------------------------------------------ + if cmr != {}: + doc.update(cmr) + #print json.dumps(cmr, indent=4) + else: + logging.debug('***>>> cmr was not set') + #------------------------------------------------------------------------------------------ + # CMR: End + #------------------------------------------------------------------------------------------ + + self.variables['docs'] = solrJson['response']['docs'] + self.variables['numFound'] = solrJson['response']['numFound'] + self.variables['itemsPerPage'] = solrJson['responseHeader']['params']['rows'] + self.variables['startIndex'] = solrJson['response']['start'] + self.variables['updated'] = datetime.datetime.utcnow().isoformat() + 'Z' + + start = int(solrJson['response']['start']) + rows = int(solrJson['responseHeader']['params']['rows']) + numFound = int(solrJson['response']['numFound']) + + logging.debug('Num docs found: ' + str(self.variables['numFound'])) + logging.debug('Items per page: ' + str(self.variables['itemsPerPage'])) + + # Show CMR statistics. + cmr_type = '' + if os.path.basename(self.variables['link']) == 'product_type': + cmr_type = 'dataset' + elif os.path.basename(self.variables['link']) == 'product': + cmr_type = 'granule' + if (cmr_type == 'dataset' or cmr_type == 'granule') and cmr_total_count != 0: + cmr_average_time = cmr_total_time/cmr_total_count + cmr_stats_msg = 'CMR search results: %s total per/%d %s(s) %s average per/%s' \ + %(cmr_total_time.__str__(), cmr_total_count, cmr_type, cmr_average_time.__str__(), cmr_type) + logging.debug(cmr_stats_msg) + + if 'facet_counts' in solrJson: + self.variables['facets'] = solrJson['facet_counts'] + + self.parameters['startIndex'] = start + self.variables['myself'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + if rows != 0: + self.parameters['startIndex'] = numFound - (numFound % rows) + self.variables['last'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + self.parameters['startIndex'] = 0 + self.variables['first'] = self.link + '?' + urllib.urlencode(self.parameters, True) + if start > 0: + if (start - rows > 0): + self.parameters['startIndex'] = start - rows + self.variables['prev'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + if start + rows < numFound: + self.parameters['startIndex'] = start + rows + self.variables['next'] = self.link + '?' + urllib.urlencode(self.parameters, True) http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/solrtemplateresponse.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/solrtemplateresponse.py b/src/main/python/libraries/edge/opensearch/solrtemplateresponse.py new file mode 100644 index 0000000..952220f --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/solrtemplateresponse.py @@ -0,0 +1,65 @@ +import datetime +import json +import logging +import urllib + +from edge.opensearch.templateresponse import TemplateResponse + +class SolrTemplateResponse(TemplateResponse): + def __init__(self, configuration, link, parameters): + super(SolrTemplateResponse, self).__init__() + self._configuration = configuration + self.link = link + self.parameters = parameters + + def generate(self, solrResponse, pretty=False): + self._populate(solrResponse) + return super(SolrTemplateResponse, self).generate(pretty) + + def _populate(self, solrResponse): + self.variables['link'] = self.link + self.variables['parameters'] = self.parameters + + start = 0 + rows = 0 + numFound = 0 + + if solrResponse is not None: + solrJson = json.loads(solrResponse) + + logging.debug('doc count: '+str(len(solrJson['response']['docs']))) + + self.variables['docs'] = solrJson['response']['docs'] + self.variables['numFound'] = solrJson['response']['numFound'] + self.variables['itemsPerPage'] = solrJson['responseHeader']['params']['rows'] + self.variables['startIndex'] = solrJson['response']['start'] + + self.variables['updated'] = datetime.datetime.utcnow().isoformat() + 'Z' + + start = int(solrJson['response']['start']) + rows = int(solrJson['responseHeader']['params']['rows']) + numFound = int(solrJson['response']['numFound']) + + logging.debug(self.variables['numFound']) + logging.debug(self.variables['itemsPerPage']) + + if 'facet_counts' in solrJson: + self.variables['facets'] = solrJson['facet_counts'] + + self.parameters['startIndex'] = start + self.variables['myself'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + if rows != 0: + self.parameters['startIndex'] = numFound - (numFound % rows) + self.variables['last'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + self.parameters['startIndex'] = 0 + self.variables['first'] = self.link + '?' + urllib.urlencode(self.parameters, True) + if start > 0: + if (start - rows > 0): + self.parameters['startIndex'] = start - rows + self.variables['prev'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + if start + rows < numFound: + self.parameters['startIndex'] = start + rows + self.variables['next'] = self.link + '?' + urllib.urlencode(self.parameters, True) http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/opensearch/templateresponse.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/opensearch/templateresponse.py b/src/main/python/libraries/edge/opensearch/templateresponse.py new file mode 100644 index 0000000..70b6211 --- /dev/null +++ b/src/main/python/libraries/edge/opensearch/templateresponse.py @@ -0,0 +1,33 @@ +import logging + +from xml.dom.minidom import * +from jinja2 import Environment, Template + +from edge.dateutility import DateUtility +from edge.opensearch.response import Response + +class TemplateResponse(Response): + def __init__(self): + super(TemplateResponse, self).__init__() + self.env = Environment() + self.env.trim_blocks = True + self.env.autoescape = True + self.variables = {} + self.env.filters['convertISOTime'] = DateUtility.convertISOTime + + def setTemplate(self, template): + self.template = self.env.from_string(template) + + def generate(self, pretty=False): + logging.debug('TemplateResponse.generate is called.') + + if pretty: + try : + xmlStr = self.template.render(self.variables).encode('utf-8').replace('\n', '') + except Exception as e: + logging.debug("Problem generating template " + str(e)) + xmlStr = self.template.render({}).encode('utf-8').replace('\n', '') + document = xml.dom.minidom.parseString(xmlStr) + return document.toprettyxml() + else: + return self.template.render(self.variables).replace('\n', '') http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/response/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/response/__init__.py b/src/main/python/libraries/edge/response/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/response/estemplateresponse.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/response/estemplateresponse.py b/src/main/python/libraries/edge/response/estemplateresponse.py new file mode 100644 index 0000000..0bb443c --- /dev/null +++ b/src/main/python/libraries/edge/response/estemplateresponse.py @@ -0,0 +1,54 @@ +import datetime +import json +import logging +import urllib + +from edge.opensearch.templateresponse import TemplateResponse + +class ESTemplateResponse(TemplateResponse): + def __init__(self, link = "", parameters = {}, defaultItemsPerPage = 0): + super(ESTemplateResponse, self).__init__() + self.link = link + self.parameters = parameters + self.defaultItemsPerPage = defaultItemsPerPage + + def generate(self, solrResponse, pretty=False): + self._populate(solrResponse) + return super(ESTemplateResponse, self).generate(pretty) + + def _populate(self, response): + start = 0 + rows = 0 + numFound = 0 + + self.variables['parameters'] = self.parameters + + if response is not None: + solrJson = json.loads(response, strict = False) + self.variables['docs'] = solrJson['hits']['hits'] + self.variables['numFound'] = int(solrJson['hits']['total']) + self.variables['itemsPerPage'] = int(self.parameters['itemsPerPage']) if 'itemsPerPage' in self.parameters else self.defaultItemsPerPage + self.variables['startIndex'] = int(self.parameters['startIndex']) if 'startIndex' in self.parameters else 0 + + start = self.variables['startIndex'] + rows = self.variables['itemsPerPage'] + numFound = self.variables['numFound'] + + + self.parameters['startIndex'] = start + self.variables['myself'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + if rows != 0: + self.parameters['startIndex'] = numFound - (numFound % rows) + self.variables['last'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + self.parameters['startIndex'] = 0 + self.variables['first'] = self.link + '?' + urllib.urlencode(self.parameters, True) + if start > 0: + if (start - rows > 0): + self.parameters['startIndex'] = start - rows + self.variables['prev'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + if start + rows < numFound: + self.parameters['startIndex'] = start + rows + self.variables['next'] = self.link + '?' + urllib.urlencode(self.parameters, True) http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/response/jsontemplateresponse.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/response/jsontemplateresponse.py b/src/main/python/libraries/edge/response/jsontemplateresponse.py new file mode 100644 index 0000000..e2da786 --- /dev/null +++ b/src/main/python/libraries/edge/response/jsontemplateresponse.py @@ -0,0 +1,33 @@ +import logging +import json + +from xml.dom.minidom import * +from jinja2 import Environment, Template + +from edge.dateutility import DateUtility +from edge.opensearch.response import Response + +class JsonTemplateResponse(Response): + def __init__(self): + super(JsonTemplateResponse, self).__init__() + self.env = Environment() + self.env.trim_blocks = True + self.env.autoescape = False + self.variables = {} + self.env.filters['convertISOTime'] = DateUtility.convertISOTime + self.env.filters['jsonify'] = self.jsonify + + def setTemplate(self, template): + self.template = self.env.from_string(template) + + def generate(self, pretty=False): + if pretty: + return json.dumps(json.loads(self.template.render(self.variables), strict=False), indent=3) + else: + return self.template.render(self.variables) + + def jsonify(self, value): + if value or value == 0: + return json.dumps(value) + else: + return "null" http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/response/solrfacettemplateresponse.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/response/solrfacettemplateresponse.py b/src/main/python/libraries/edge/response/solrfacettemplateresponse.py new file mode 100644 index 0000000..40c7e95 --- /dev/null +++ b/src/main/python/libraries/edge/response/solrfacettemplateresponse.py @@ -0,0 +1,23 @@ +import json +import logging + +from edge.response.jsontemplateresponse import JsonTemplateResponse + +class SolrFacetTemplateResponse(JsonTemplateResponse): + def __init__(self, facetDefs): + super(SolrFacetTemplateResponse, self).__init__() + self.facetDefs = facetDefs + + def generate(self, solrResponse, pretty=False): + self._populate(solrResponse) + return super(SolrFacetTemplateResponse, self).generate(pretty) + + def _populate(self, solrResponse): + if solrResponse is not None: + solrJson = json.loads(solrResponse) + + logging.debug('doc count: '+str(len(solrJson['response']['docs']))) + + if 'facet_counts' in solrJson: + self.variables['facets'] = solrJson['facet_counts'] + self.variables['facetDefs'] = self.facetDefs http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/response/solrjsontemplateresponse.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/response/solrjsontemplateresponse.py b/src/main/python/libraries/edge/response/solrjsontemplateresponse.py new file mode 100644 index 0000000..76d988f --- /dev/null +++ b/src/main/python/libraries/edge/response/solrjsontemplateresponse.py @@ -0,0 +1,60 @@ +import datetime +import json +import logging +import urllib + +from edge.response.jsontemplateresponse import JsonTemplateResponse + +class SolrJsonTemplateResponse(JsonTemplateResponse): + def __init__(self, link = "", parameters = {}): + super(SolrJsonTemplateResponse, self).__init__() + self.link = link + self.parameters = parameters + + def generate(self, solrResponse, pretty=False): + self._populate(solrResponse) + return super(SolrJsonTemplateResponse, self).generate(pretty) + + def _populate(self, solrResponse): + start = 0 + rows = 0 + numFound = 0 + + self.variables['parameters'] = self.parameters + + if solrResponse is not None: + solrJson = json.loads(solrResponse, strict = False) + + self.variables['docs'] = solrJson['response']['docs'] + self.variables['numFound'] = int(solrJson['response']['numFound']) + self.variables['itemsPerPage'] = int(solrJson['responseHeader']['params']['rows']) + self.variables['startIndex'] = int(solrJson['response']['start']) + + if 'stats' in solrJson: + self.variables['stats'] = solrJson['stats'] + + if 'facet_counts' in solrJson: + self.variables['facets'] = solrJson['facet_counts'] + + start = int(solrJson['response']['start']) + rows = int(solrJson['responseHeader']['params']['rows']) + numFound = int(solrJson['response']['numFound']) + + + self.parameters['startIndex'] = start + self.variables['myself'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + if rows != 0: + self.parameters['startIndex'] = numFound - (numFound % rows) + self.variables['last'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + self.parameters['startIndex'] = 0 + self.variables['first'] = self.link + '?' + urllib.urlencode(self.parameters, True) + if start > 0: + if (start - rows > 0): + self.parameters['startIndex'] = start - rows + self.variables['prev'] = self.link + '?' + urllib.urlencode(self.parameters, True) + + if start + rows < numFound: + self.parameters['startIndex'] = start + rows + self.variables['next'] = self.link + '?' + urllib.urlencode(self.parameters, True) http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/spatialsearch.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/spatialsearch.py b/src/main/python/libraries/edge/spatialsearch.py new file mode 100644 index 0000000..0bfa27e --- /dev/null +++ b/src/main/python/libraries/edge/spatialsearch.py @@ -0,0 +1,66 @@ +import sys +import logging +#import cx_Oracle + +class SpatialSearch(object): + def __init__(self, identity): + self.identity = identity + logging.debug('identity: '+self.identity) + + def searchGranules(self, offset, rows, west, south, east, north): + logging.debug('w='+str(west)+',s='+str(south)+',e='+str(east)+',n='+str(north)) + + ids = [] + resultCount = 0 + connection = None + ''' + try: + connection = cx_Oracle.connect(self.identity) + cursor = connection.cursor() + refCursor = connection.cursor() + + westValue = cursor.var(cx_Oracle.NUMBER) + westValue.setvalue(0, west) + southValue = cursor.var(cx_Oracle.NUMBER) + southValue.setvalue(0, south) + eastValue = cursor.var(cx_Oracle.NUMBER) + eastValue.setvalue(0, east) + northValue = cursor.var(cx_Oracle.NUMBER) + northValue.setvalue(0, north) + offsetValue = cursor.var(cx_Oracle.NUMBER) + offsetValue.setvalue(0, offset + 1) + rowsValue = cursor.var(cx_Oracle.NUMBER) + rowsValue.setvalue(0, rows) + + cursor.execute( + 'select inventory.searchGranuleSpatialCount(:south,:west,:north,:east) FROM dual', + {'south': southValue, 'west': westValue, 'north': northValue, 'east': eastValue} + ) + result = cursor.fetchone() + if result is None: + raise Exception('Failed to get count from inventory.searchGranuleSpatialCount.') + else: + resultCount = int(result[0]) + + cursor.callproc( + 'Inventory.searchGSpatial', + [southValue, westValue, northValue, eastValue, offsetValue, rowsValue, refCursor] + ) + + logging.debug('rowcount: '+str(cursor.rowcount)) + logging.debug('ref rowcount: '+str(refCursor.rowcount)) + + row = refCursor.next() + while row: + ids.append(row[0]) + row = refCursor.next() + except StopIteration: + pass + except BaseException as detail: + print 'ouch', detail + logging.error('Failed to search granules: '+str(sys.exc_info()[0])) + finally: + if connection is not None: + connection.close() + ''' + return (ids, resultCount) http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/writer/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/writer/__init__.py b/src/main/python/libraries/edge/writer/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/writer/estemplateresponsewriter.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/writer/estemplateresponsewriter.py b/src/main/python/libraries/edge/writer/estemplateresponsewriter.py new file mode 100644 index 0000000..e947031 --- /dev/null +++ b/src/main/python/libraries/edge/writer/estemplateresponsewriter.py @@ -0,0 +1,116 @@ +from types import * +import logging +import urllib +import json +from collections import OrderedDict + +from edge.httputility import HttpUtility +from edge.writer.templateresponsewriter import TemplateResponseWriter + +class ESTemplateResponseWriter(TemplateResponseWriter): + def __init__(self, configFilePath, requiredParams = None): + super(ESTemplateResponseWriter, self).__init__(configFilePath, requiredParams) + self.searchParameters = {} + self.variables = {} + self.facet = False + self.facetDefs = {} + self.contentType = 'application/xml' + + def get(self, requestHandler): + super(ESTemplateResponseWriter, self).get(requestHandler) + + startIndex = 0 + try: + startIndex = requestHandler.get_argument('startIndex') + self.searchParameters['startIndex'] = startIndex + except: + pass + + entriesPerPage = self._configuration.getint('solr', 'entriesPerPage') + try: + entriesPerPage = requestHandler.get_argument('itemsPerPage') + maxEntriesPerPage = self._configuration.getint('solr', 'maxEntriesPerPage') + if (int(entriesPerPage) > maxEntriesPerPage): + entriesPerPage = maxEntriesPerPage + self.searchParameters['itemsPerPage'] = entriesPerPage + except: + pass + + try: + if requestHandler.get_argument('pretty').lower() == 'true': + self.pretty = True + self.searchParameters['pretty'] = 'true' + except: + pass + + parameters = {} + for parameter in self._configuration.get('solr', 'parameters').split(','): + try: + value = requestHandler.get_arguments(parameter) + if len(value) == 1: + parameters[parameter] = value[0] + self.searchParameters[parameter] = value[0] + elif len(value) > 0: + parameters[parameter] = value + self.searchParameters[parameter] = value + except: + pass + + facets = {} + if self._configuration.has_option('solr', 'facets'): + self.facetDefs = json.loads(self._configuration.get('solr', 'facets'), object_pairs_hook=OrderedDict) + for facet in self.facetDefs.keys(): + try: + value = requestHandler.get_arguments(facet) + if len(value) > 0: + facets[self.facetDefs[facet]] = value + self.searchParameters[facet] = value + except: + pass + + try: + self._getResponse(startIndex, entriesPerPage, parameters, facets) + except: + logging.exception('Failed to get solr response.') + + def _urlEncodeSolrQueryValue(self, value): + return urllib.quote('"'+value+'"') + + def _onResponse(self, response): + logging.debug(response) + if response.error: + self._handleException(str(response.error)) + else: + self._writeResponse(response.body) + + def _writeResponse(self, responseText): + searchText = '' + if 'keyword' in self.variables: + searchText = self.variables['keyword'] + try: + openSearchResponse = self._generateOpenSearchResponse( + responseText, + searchText, + self._configuration.get('service', 'url') + self.requestHandler.request.path, + self.searchParameters, + self.pretty + ) + self.requestHandler.set_header("Content-Type", self.contentType) + self.requestHandler.set_header('Access-Control-Allow-Origin', '*') + self.requestHandler.write(openSearchResponse) + self.requestHandler.finish() + except BaseException as exception: + self._handleException(str(exception)) + + def _getResponse(self, startIndex, entriesPerPage, parameters, facets): + query = self._constructQuery(startIndex, entriesPerPage, parameters, facets) + url = self._configuration.get('solr', 'datasetUrl') + + httpUtility = HttpUtility() + httpUtility.getResponse(url+'/_search/?'+query, self._onResponse) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + pass + + def _constructQuery(self, startIndex, entriesPerPage, parameters, facets): + pass http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/writer/genericproxywriter.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/writer/genericproxywriter.py b/src/main/python/libraries/edge/writer/genericproxywriter.py new file mode 100644 index 0000000..dd7b1da --- /dev/null +++ b/src/main/python/libraries/edge/writer/genericproxywriter.py @@ -0,0 +1,14 @@ +import logging +import urllib + +from edge.writer.proxywriter import ProxyWriter + +class GenericProxyWriter(ProxyWriter): + def __init__(self, configFilePath): + super(GenericProxyWriter, self).__init__(configFilePath) + + def _generateUrl(self, requestHandler): + url = self._configuration.get('proxy', 'url') + url += requestHandler.request.uri[requestHandler.request.uri.index("?"):] + logging.debug("proxy to url : " + url) + return url http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/writer/proxywriter.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/writer/proxywriter.py b/src/main/python/libraries/edge/writer/proxywriter.py new file mode 100644 index 0000000..f747681 --- /dev/null +++ b/src/main/python/libraries/edge/writer/proxywriter.py @@ -0,0 +1,32 @@ +import logging + +import requestresponder +from edge.httputility import HttpUtility + +class ProxyWriter(requestresponder.RequestResponder): + def __init__(self, configFilePath): + super(ProxyWriter, self).__init__(configFilePath) + + def get(self, requestHandler): + super(ProxyWriter, self).get(requestHandler) + try: + httpUtility = HttpUtility() + result = httpUtility.getResponse(self._generateUrl(requestHandler), self.onResponse) + except BaseException as exception: + raise exception + + def onResponse(self, response): + if response.error: + self.requestHandler.set_status(404) + self.requestHandler.write(str(response.error)) + self.requestHandler.finish() + else: + for name, value in response.headers.iteritems(): + logging.debug('header: '+name+':'+value) + self.requestHandler.set_header(name, value) + self.requestHandler.set_header('Access-Control-Allow-Origin', '*') + self.requestHandler.write(response.body) + self.requestHandler.finish() + + def _generateUrl(self, requestHandler): + pass http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/writer/solrtemplateresponsewriter.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/writer/solrtemplateresponsewriter.py b/src/main/python/libraries/edge/writer/solrtemplateresponsewriter.py new file mode 100644 index 0000000..636a21a --- /dev/null +++ b/src/main/python/libraries/edge/writer/solrtemplateresponsewriter.py @@ -0,0 +1,115 @@ +from types import * +import logging +import urllib +import json +from collections import OrderedDict + +from edge.httputility import HttpUtility +from edge.writer.templateresponsewriter import TemplateResponseWriter + +class SolrTemplateResponseWriter(TemplateResponseWriter): + def __init__(self, configFilePath, requiredParams = None): + super(SolrTemplateResponseWriter, self).__init__(configFilePath, requiredParams) + self.searchParameters = {} + self.variables = {} + self.facet = False + self.facetDefs = {} + self.contentType = 'application/xml' + + def get(self, requestHandler): + super(SolrTemplateResponseWriter, self).get(requestHandler) + + startIndex = 0 + try: + startIndex = requestHandler.get_argument('startIndex') + except: + pass + + entriesPerPage = self._configuration.getint('solr', 'entriesPerPage') + try: + entriesPerPage = requestHandler.get_argument('itemsPerPage') + maxEntriesPerPage = self._configuration.getint('solr', 'maxEntriesPerPage') + if (int(entriesPerPage) > maxEntriesPerPage): + entriesPerPage = maxEntriesPerPage + self.searchParameters['itemsPerPage'] = entriesPerPage + except: + pass + + try: + if requestHandler.get_argument('pretty').lower() == 'true': + self.pretty = True + self.searchParameters['pretty'] = 'true' + except: + pass + + parameters = {} + for parameter in self._configuration.get('solr', 'parameters').split(','): + try: + value = requestHandler.get_arguments(parameter) + if len(value) == 1: + parameters[parameter] = value[0] + self.searchParameters[parameter] = value[0] + elif len(value) > 0: + parameters[parameter] = value + self.searchParameters[parameter] = value + except: + pass + + facets = {} + if self._configuration.has_option('solr', 'facets'): + self.facetDefs = json.loads(self._configuration.get('solr', 'facets'), object_pairs_hook=OrderedDict) + for facet in self.facetDefs.keys(): + try: + value = requestHandler.get_arguments(facet) + if len(value) > 0: + facets[self.facetDefs[facet]] = value + self.searchParameters[facet] = value + except: + pass + + try: + self._getSolrResponse(startIndex, entriesPerPage, parameters, facets) + except: + logging.exception('Failed to get solr response.') + + def _urlEncodeSolrQueryValue(self, value): + return urllib.quote('"'+value+'"') + + def _onSolrResponse(self, response): + logging.debug(response) + if response.error: + self._handleException(str(response.error)) + else: + self._writeResponse(response.body) + + def _writeResponse(self, responseText): + searchText = '' + if 'keyword' in self.variables: + searchText = self.variables['keyword'] + try: + openSearchResponse = self._generateOpenSearchResponse( + responseText, + searchText, + self._configuration.get('service', 'url') + self.requestHandler.request.path, + self.searchParameters, + self.pretty + ) + self.requestHandler.set_header("Content-Type", self.contentType) + self.requestHandler.set_header('Access-Control-Allow-Origin', '*') + self.requestHandler.write(openSearchResponse) + self.requestHandler.finish() + except BaseException as exception: + self._handleException(str(exception)) + + def _getSolrResponse(self, startIndex, entriesPerPage, parameters, facets): + query = self._constructSolrQuery(startIndex, entriesPerPage, parameters, facets) + url = self._configuration.get('solr', 'datasetUrl') + + httpUtility = HttpUtility() + httpUtility.getResponse(url+'/select/?'+query, self._onSolrResponse) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + pass + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + pass http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/libraries/edge/writer/templateresponsewriter.py ---------------------------------------------------------------------- diff --git a/src/main/python/libraries/edge/writer/templateresponsewriter.py b/src/main/python/libraries/edge/writer/templateresponsewriter.py new file mode 100644 index 0000000..17b9abc --- /dev/null +++ b/src/main/python/libraries/edge/writer/templateresponsewriter.py @@ -0,0 +1,40 @@ +from types import * +import logging +import codecs + +import requestresponder +from edge.httputility import HttpUtility + +class TemplateResponseWriter(requestresponder.RequestResponder): + def __init__(self, configFilePath, requiredParams = None): + super(TemplateResponseWriter, self).__init__(configFilePath) + if requiredParams is None: + requiredParams = [] + self.requiredParams = requiredParams + self.pretty = False + + def get(self, requestHandler): + super(TemplateResponseWriter, self).get(requestHandler) + + #check required parameters + for paramList in self.requiredParams: + countParamNotFound = 0 + for param in paramList: + try: + requestHandler.get_argument(param) + except: + countParamNotFound += 1 + if countParamNotFound == len(paramList): + raise Exception("One of the following parameters is required: " + ', '.join(paramList)) + + def _handleException(self, error): + self.requestHandler.set_status(404) + self.requestHandler.write(error) + self.requestHandler.finish() + + def _readTemplate(self, path): + file = codecs.open(path, encoding='utf-8') + data = file.read() + file.close() + + return data http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/logging.conf ---------------------------------------------------------------------- diff --git a/src/main/python/logging.conf b/src/main/python/logging.conf new file mode 100644 index 0000000..de2f611 --- /dev/null +++ b/src/main/python/logging.conf @@ -0,0 +1,28 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler,timedRotatingFileHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler,timedRotatingFileHandler + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=simpleFormatter +args=(sys.stdout,) + +[handler_timedRotatingFileHandler] +class=handlers.TimedRotatingFileHandler +level=ERROR +formatter=simpleFormatter +args=(r'./tornado.log', 'midnight', 1, 30) + +[formatter_simpleFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/pluginhandler.py ---------------------------------------------------------------------- diff --git a/src/main/python/pluginhandler.py b/src/main/python/pluginhandler.py new file mode 100644 index 0000000..0b72788 --- /dev/null +++ b/src/main/python/pluginhandler.py @@ -0,0 +1,58 @@ +import os +import sys +import logging + +class PluginHandler(object): + def __init__(self, name, pluginPath, format=None): + self.name = name + self.pluginPath = pluginPath + self.format = format + + def handleRequest(self, httpMethod, path, requestHandler): + paths = path.split('/') + fileName = paths.pop() + fileNames = fileName.split('.') + fileSuffix = fileNames.pop() + + # Support plugin lookup for [fgdc|gmcd|iso].xml + if fileSuffix == 'xml': + fileSuffix = fileNames.pop() + + if self.format is not None and len(self.format) > 0: + try: + fileSuffix = requestHandler.get_argument('format') + except: + fileSuffix = self.format[0] + #raise Exception("Format parameter required.") + if fileSuffix not in self.format: + raise Exception("Format %s not supported." % fileSuffix) + + pluginName = self._getPluginName(self.pluginPath+'/'+self.name+'/'+fileSuffix) + if not pluginName: + raise Exception("Did not find plugin.") + + modulePath = self.pluginPath+'.'+self.name+'.'+fileSuffix+'.'+pluginName + if modulePath in sys.modules: + currentModuleName = '' + for moduleName in modulePath.split('.'): + currentModuleName += moduleName + #print('reloading: '+currentModuleName) + reload(sys.modules[currentModuleName]) + currentModuleName += '.' + + #print('modulePath: '+modulePath) + module = __import__(modulePath, globals(), locals(), [pluginName]) + plugin = getattr(module, pluginName) + pluginObject = plugin(self.pluginPath+'/'+self.name+'/'+fileSuffix+'/plugin.conf') + method = getattr(pluginObject, httpMethod) + method(requestHandler) + + def _getPluginName(self, path): + name = None + for fileName in os.listdir(path): + if fileName != '__init__.py' and fileName.endswith('.py'): + name = fileName.split('.')[0] + break + + return name + http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/TestPlugin.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/TestPlugin.py b/src/main/python/plugins/TestPlugin.py new file mode 100644 index 0000000..8ddc657 --- /dev/null +++ b/src/main/python/plugins/TestPlugin.py @@ -0,0 +1,5 @@ +#import tornado.web + +class TestPlugin: + def run(self): + print("aaa") http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/__init__.py b/src/main/python/plugins/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/dataset/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/dataset/__init__.py b/src/main/python/plugins/dataset/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/dataset/atom/AtomWriter.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/dataset/atom/AtomWriter.py b/src/main/python/plugins/dataset/atom/AtomWriter.py new file mode 100644 index 0000000..3995e39 --- /dev/null +++ b/src/main/python/plugins/dataset/atom/AtomWriter.py @@ -0,0 +1,26 @@ +import logging +import datetime + +from edge.opensearch.datasetatomresponse import DatasetAtomResponse +from edge.opensearch.datasetwriter import DatasetWriter + +class AtomWriter(DatasetWriter): + def __init__(self, configFilePath): + super(AtomWriter, self).__init__(configFilePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = DatasetAtomResponse( + self._configuration.get('portal', 'datasetUrl'), + self._configuration.get('service', 'host'), + self._configuration.get('service', 'url'), + self.datasets + ) + + response.title = 'PO.DAAC Dataset Search Results' + response.link = searchUrl + response.authors.append('PO.DAAC Dataset Search Service') + response.updated = datetime.datetime.utcnow().isoformat()+'Z' + response.id = 'tag:'+self._configuration.get('service', 'host')+','+datetime.datetime.utcnow().date().isoformat() + response.parameters = searchParams + + return response.generate(solrResponse, pretty) http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/dataset/atom/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/dataset/atom/__init__.py b/src/main/python/plugins/dataset/atom/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/dataset/atom/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/dataset/atom/plugin.conf b/src/main/python/plugins/dataset/atom/plugin.conf new file mode 100644 index 0000000..ae94729 --- /dev/null +++ b/src/main/python/plugins/dataset/atom/plugin.conf @@ -0,0 +1,11 @@ +[solr] +datasetUrl=http://localhost:8983/solr.war/dataset +granuleUrl=http://localhost:8983/solr.war/granule +entriesPerPage=7 + +[portal] +datasetUrl=http://localhost:8000/drupal/dataset + +[service] +url=http://localhost:8890 +host=localhost:8890 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/dataset/gcmd/DifWriter.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/dataset/gcmd/DifWriter.py b/src/main/python/plugins/dataset/gcmd/DifWriter.py new file mode 100644 index 0000000..407dda3 --- /dev/null +++ b/src/main/python/plugins/dataset/gcmd/DifWriter.py @@ -0,0 +1,32 @@ +import logging +import os +import os.path +import codecs + +from edge.opensearch.datasetgcmdresponse import DatasetGcmdResponse +from edge.opensearch.datasetwriter import DatasetWriter + +class DifWriter(DatasetWriter): + def __init__(self, configFilePath): + super(DifWriter, self).__init__(configFilePath) + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = DatasetGcmdResponse(self._configuration) + response.setTemplate(self.template) + + allowNone = False + if 'allowNone' in searchParams and searchParams['allowNone'].lower() == 'true': + allowNone = True + + return response.generate(solrResponse, pretty=pretty, allowNone=allowNone) + + def _readTemplate(self, path): + file = codecs.open(path, encoding='utf-8') + data = file.read() + file.close() + + return data http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/dataset/gcmd/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/dataset/gcmd/__init__.py b/src/main/python/plugins/dataset/gcmd/__init__.py new file mode 100644 index 0000000..e69de29
