http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/content/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/content/Writer.py b/src/main/python/plugins/slcp/content/Writer.py new file mode 100644 index 0000000..c0e27ab --- /dev/null +++ b/src/main/python/plugins/slcp/content/Writer.py @@ -0,0 +1,76 @@ +import logging +import os +import os.path +import urllib + +from edge.writer.solrtemplateresponsewriter import SolrTemplateResponseWriter +from edge.response.solrjsontemplateresponse import SolrJsonTemplateResponse + +class Writer(SolrTemplateResponseWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + self.contentType = 'application/json' + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = SolrJsonTemplateResponse() + response.setTemplate(self.template) + + return response.generate(solrResponse, pretty=pretty) + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + queries = [] + filterQueries = [] + filterQueries.append('status:1') + sort = None + + for key, value in parameters.iteritems(): + if value != "": + if key == 'keyword': + #Special case keyword search on glossary_items only match title + if 'table' in parameters and parameters['table'] == 'glossary_items': + queries.append('title_t:('+urllib.quote(value) + ')') + else: + queries.append(urllib.quote(value)) + elif key == 'year': + start = value + "-01-01T00:00:00.000Z" + end = value + "-12-31T23:59:59.999Z" + filterQueries.append('created_at:['+start+'%20TO%20'+end+']') + elif key == 'table': + filterQueries.append('type:' + value) + elif key == 'glossary_title': + range = value.lower().split('-') + filterQueries.append('{!frange%20l=' + range[0] + '%20u=' + range[1] + 'z}' + 'title_lc') + elif key == 'sort': + sort = urllib.quote(value) + elif key == 'topic_id': + filterQueries.append('categories_id:' + value) + elif key == 'mission_id': + filterQueries.append('mission_ids_array:' + value) + else: + if type(value) is list: + if 'table' in parameters and parameters['table'] == 'news_items': + filterQueries.append(key + ':(' + '+OR+'.join([self._urlEncodeSolrQueryValue(v) for v in value]) + ')') + else: + for v in value: + filterQueries.append(key + ':' + self._urlEncodeSolrQueryValue(v)) + else: + filterQueries.append(key + ':' + self._urlEncodeSolrQueryValue(value)) + if len(queries) == 0: + queries.append('*:*') + + query = 'q='+'+AND+'.join(queries)+'&version=2.2&indent=on&wt=json&start='+str(startIndex)+'&rows='+str(entriesPerPage) + + if len(filterQueries) > 0: + query += '&fq='+'+AND+'.join(filterQueries) + + if sort is not None: + query += '&sort=' + sort + + logging.debug('solr query: '+query) + + return query
http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/content/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/content/__init__.py b/src/main/python/plugins/slcp/content/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/content/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/content/plugin.conf b/src/main/python/plugins/slcp/content/plugin.conf new file mode 100644 index 0000000..3e74010 --- /dev/null +++ b/src/main/python/plugins/slcp/content/plugin.conf @@ -0,0 +1,11 @@ +[solr] +datasetUrl=http://localhost:8983/solr/content +entriesPerPage=10 +maxEntriesPerPage=2000 +defaultSearchParam=keyword +parameters=keyword,sort,categories_id,status,is_in_resource_list,category_type_id,is_faq,table,featured,year,publication_year,glossary_title,id,topic_id,mission_id +facets={} + +[service] +url=http://localhost:8890 +template=template.xml http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/content/template.xml ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/content/template.xml b/src/main/python/plugins/slcp/content/template.xml new file mode 100755 index 0000000..357ff59 --- /dev/null +++ b/src/main/python/plugins/slcp/content/template.xml @@ -0,0 +1,158 @@ +{"items":[ +{% for doc in docs %} +{ +{% if doc['type'] == 'resources' %} + "alt_text": {{ doc['alt_text'] | jsonify }}, + "created_at": {{ doc['created_at'] | jsonify }}, + "credit": {{ doc['credit'] | jsonify }}, + "detail_content_type": {{ doc['detail_content_type'] | jsonify }}, + "detail_file_name": {{ doc['detail_file_name'] | jsonify }}, + "detail_file_size": {{ doc['detail_file_size'] | jsonify }}, + "detail_image": {{ doc['detail_image'] | jsonify }}, + "embed_code": {{ doc['embed_code'] | jsonify }}, + "external_url": {{ doc['external_url'] | jsonify }}, + "featured": {{ doc['featured'] | jsonify }}, + "id": {{ doc['id'] }}, + "is_in_resource_list": {{ doc['is_in_resource_list'] | jsonify }}, + "is_latest": {{ doc['is_latest'] | jsonify }}, + "latest_image_file_name": {{ doc['latest_image_file_name'] | jsonify }}, + "list_image_content_type": {{ doc['list_image_content_type'] | jsonify }}, + "list_image_file_name": {{ doc['list_image_file_name'] | jsonify }}, + "list_image_file_size": {{ doc['list_image_file_size'] | jsonify }}, + "list_image_src": "/system/resources/list_images/{{ doc['id'] }}_{{ doc['list_image_file_name'] }}", + "long_description": {{ doc['long_description'] | jsonify }}, + "pub_date": {{ doc['pub_date'] | jsonify }}, + "short_description": {{ doc['short_description'] | jsonify }}, + "status": {{ doc['status'] }}, + "table": {{ doc['type'] | jsonify }}, + "title": {{ doc['title'] | jsonify }}, + "updated_at": {{ doc['updated_at'] | jsonify }}, + "updated_by": {{ doc['updated_by'] | jsonify }}, + "url": {{ doc['permalink_path'] | jsonify }} +{% elif doc['type'] == 'publications' %} + "abstract": {{ doc['abstract'] | jsonify }}, + "author_address": {{ doc['author_address'] | jsonify }}, + "authors": {{ doc['authors'] | jsonify }}, + "category": {{ doc['category'] | jsonify }}, + "citation": {{ doc['citation'] | jsonify }}, + "created_at": {{ doc['created_at'] | jsonify }}, + "external_uid": {{ doc['external_uid'] | jsonify }} , + "file_content_type": {{ doc['file_content_type'] | jsonify }}, + "file_file_name": {{ doc['file_file_name'] | jsonify }}, + "file_file_size": {{ doc['file_file_size'] | jsonify }}, + "graph_category": {{ doc['graph_category'] | jsonify }}, + "id": {{ doc['id'] }}, + "is_peer_reviewed": {{ doc['is_peer_reviewed'] | lower }}, + "keywords": {{ doc['keywords'] | jsonify }}, + "link": {{ doc['link'] | jsonify }}, + "periodical": { + "created_at": {{ doc['periodical_created_at'] | jsonify }}, + "id": {{ doc['periodical_id'] }}, + "title": {{ doc['periodical_title'] | jsonify }}, + "updated_at": {{ doc['periodical_updated_at'] | jsonify }}, + "updated_by": {{ doc['periodical_updated_by'] | jsonify }} + }, + "periodical_id": {{ doc['periodical_id'] }}, + "publication_year": {{ doc['publication_year'] | jsonify }}, + "status": {{ doc['status'] }}, + "table": {{ doc['type'] | jsonify }}, + "title": {{ doc['title'] | jsonify }}, + "updated_at": {{ doc['updated_at'] | jsonify }}, + "updated_by": {{ doc['updated_by'] | jsonify }}, + "url": {{ doc['permalink_path'] | jsonify }} +{% elif doc['type'] == 'news_items' %} + "categories_id": {{ doc['categories_id'] | jsonify }}, + "categories_title": {{ doc['categories_title'] | jsonify }}, + "date": "{{ doc['publish_date'] | convertISOTime('%B %-d, %Y') }}", + "description": {{ doc['long_description'] | jsonify }}, + "id": {{ doc['id']}}, + "main_image_alt": {{ doc['main_image_alt'] | jsonify }}, + "main_image_caption": {{ doc['main_image_caption'] | jsonify }}, + "main_image_file_name": {{ doc['main_image_file_name'] | jsonify }}, + "status": {{ doc['status'] }}, + "table": {{ doc['type'] | jsonify }}, + "target": "_self", + "thumb": "/system/news_items/list_view_images/{{ doc['id']}}_{{ doc['list_view_image_file_name']}}", + "title": {{ doc['title'] | jsonify }}, + "url": {{ doc['permalink_path'] | jsonify }}, + "mission_ids": {{ doc['mission_ids_array'] | jsonify }} +{% elif doc['type'] == 'glossary_items' %} + "caption": {{ doc['caption'] | jsonify }}, + "created_at": {{ doc['created_at'] | jsonify }}, + "credit": {{ doc['credit'] | jsonify }}, + "definition": {{ doc['definition'] | jsonify }}, + "id": {{ doc['id']}}, + "list_view_image_alt": {{ doc['list_view_image_alt'] | jsonify }}, + "list_view_image_file_name": {{ doc['list_view_image_file_name'] | jsonify }}, + "status": {{ doc['status'] }}, + "table": {{ doc['type'] | jsonify }}, + "title": {{ doc['title'] | jsonify }}, + "updated_at": {{ doc['updated_at'] | jsonify }}, + "url": {{ doc['permalink_path'] | jsonify }}, + "video_embed_code": {{ doc['video_embed_code'] | jsonify }} +{% elif doc['type'] == 'content_pages' %} + "body": {{ doc['body'] | jsonify }}, + "created_at": {{ doc['created_at'] | jsonify }}, + "id": {{ doc['id']}}, + "main_image_alt": {{ doc['main_image_alt'] | jsonify }}, + "main_image_caption": {{ doc['main_image_caption'] | jsonify }}, + "main_image_file_name": {{ doc['main_image_file_name'] | jsonify }}, + "meta_desc": {{ doc['meta_desc'] | jsonify }}, + "status": {{ doc['status'] }}, + "subnav_title": {{ doc['subnav_title'] | jsonify }}, + "table": {{ doc['type'] | jsonify }}, + "title": {{ doc['title'] | jsonify }}, + "updated_at": {{ doc['updated_at'] | jsonify }}, + "updated_by": {{ doc['updated_by'] | jsonify }}, + "url": {{ doc['permalink_path'] | jsonify }} +{% elif doc['type'] == 'faq_items' %} + "answer": {{ doc['answer'] | jsonify }}, + "created_at": {{ doc['created_at'] | jsonify }}, + "data_release_version": {{ doc['data_release_version'] | jsonify }}, + "id": {{ doc['id']}}, + "is_faq": {{ doc['is_faq'] | jsonify }}, + "keywords": {{ doc['keywords'] | jsonify }}, + "physical_product": {{ doc['physical_product'] | jsonify }}, + "post_date": {{ doc['post_date'] | jsonify }}, + "question": {{ doc['question'] | jsonify }}, + "question_type": {{ doc['question_type'] | jsonify }}, + "sent_by": {{ doc['sent_by'] | jsonify }}, + "status": {{ doc['status'] }}, + "subject": {{ doc['subject'] | jsonify }}, + "table": {{ doc['type'] | jsonify }}, + "updated_at": {{ doc['updated_at'] | jsonify }}, + "updated_by": {{ doc['updated_by'] | jsonify }}, + "url": {{ doc['permalink_path'] | jsonify }} +{% elif doc['type'] == 'missions' %} + "start_date": {{ doc['start_date'] | jsonify }}, + "end_date": {{ doc['end_date'] | jsonify }}, + "title": {{ doc['title'] | jsonify }}, + "id": {{ doc['id']}}, + "url": {{ doc['permalink_path'] | jsonify }}, + "short_description": {{ doc['short_description'] | jsonify }}, + "body": {{ doc['body'] | jsonify }}, + "image_file_name": {{ doc['image_file_name'] | jsonify }}, + "list_view_image_file_name": {{ doc['list_view_image_file_name'] | jsonify }}, + "list_view_image_alt": {{ doc['list_view_image_alt'] | jsonify }}, + "image_alt": {{ doc['image_alt'] | jsonify }}, + "status": {{ doc['status'] }}, + "created_at": {{ doc['created_at'] | jsonify }}, + "short_title": {{ doc['short_title'] | jsonify }}, + "updated_at": {{ doc['updated_at'] | jsonify }}, + "sidebar_body": {{ doc['sidebar_body'] | jsonify }} +{% endif %} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +,"more": +{% if (startIndex + itemsPerPage) < numFound %} +true +{% else %} +false +{% endif %} +,"total":{{ numFound }} +,"page":{{ (startIndex / itemsPerPage + 1) | int }} +,"per_page":{{ itemsPerPage }}} http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/dat/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/dat/Writer.py b/src/main/python/plugins/slcp/dat/Writer.py new file mode 100644 index 0000000..1bdfe55 --- /dev/null +++ b/src/main/python/plugins/slcp/dat/Writer.py @@ -0,0 +1,53 @@ +import logging +import os +import os.path +import urllib + +from edge.writer.solrtemplateresponsewriter import SolrTemplateResponseWriter +from edge.response.solrjsontemplateresponse import SolrJsonTemplateResponse + +class Writer(SolrTemplateResponseWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + self.contentType = 'application/json' + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = SolrJsonTemplateResponse() + response.setTemplate(self.template) + + return response.generate(solrResponse, pretty=pretty) + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + queries = [] + filterQueries = [] + + for key, value in parameters.iteritems(): + if key == 'id': + queries.append('id:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'slcpShortName': + queries.append('SlcpShortName:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'shortName': + queries.append('ShortName:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'nexusShortName': + queries.append('GlobalAttrNexusShortName:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'inDAT': + filterQueries.append('InDAT:%s' % value) + + if len(queries) == 0: + queries.append('*:*') + + query = 'q='+'+AND+'.join(queries)+'&version=2.2&indent=on&wt=json'+'&rows='+str(entriesPerPage) + + if len(filterQueries) > 0: + query += '&fq='+'+AND+'.join(filterQueries) + + query += '&sort=' + urllib.quote("DATOrder desc,ShortName asc") + + logging.debug('solr query: '+query) + + return query http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/dat/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/dat/__init__.py b/src/main/python/plugins/slcp/dat/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/dat/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/dat/plugin.conf b/src/main/python/plugins/slcp/dat/plugin.conf new file mode 100644 index 0000000..2c69d4f --- /dev/null +++ b/src/main/python/plugins/slcp/dat/plugin.conf @@ -0,0 +1,9 @@ +[solr] +datasetUrl=http://localhost:8983/solr/dataset +entriesPerPage=10 +maxEntriesPerPage=2000 +parameters=id,slcpShortName,shortName,nexusShortName,inDAT + +[service] +url=http://localhost:8890 +template=template.json http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/dat/template.json ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/dat/template.json b/src/main/python/plugins/slcp/dat/template.json new file mode 100644 index 0000000..f7e96ae --- /dev/null +++ b/src/main/python/plugins/slcp/dat/template.json @@ -0,0 +1,33 @@ +{ +"Datasets": [ +{% for doc in docs %} + +{% for i in range(doc['GlobalAttrTitle']|count) %} +{ +"GlobalAttrShortName": "{{ doc['ShortName'] }}", +"GlobalAttrNexusShortName": "{{ doc['GlobalAttrNexusShortName'][i] }}", +"GlobalAttrTitle": "{{ doc['GlobalAttrTitle'][i] }}", +"GlobalAttrDescription": "{{ doc['GlobalAttrDescription'][i] }}", +"GlobalAttrSource": "{{ doc['GlobalAttrSource'][i] }}", +"GlobalAttrContact": "{{ doc['GlobalAttrContact'][i] }}", +"GlobalAttrUnits": "{{ doc['GlobalAttrUnits'][i] }}" +{% if 'SupportsBasin' in doc %} +,"SupportsBasin": {{ doc['SupportsBasin'] | jsonify }} +{% endif %} +{% if 'Climatology' in doc %} +,"Climatology": {{ doc['Climatology'] | jsonify }} +{% endif %} +{% if 'IsClimatology' in doc %} +,"IsClimatology": {{ doc['IsClimatology'] | jsonify }} +{% endif %} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/echo10/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/echo10/Writer.py b/src/main/python/plugins/slcp/echo10/Writer.py new file mode 100644 index 0000000..ebafc40 --- /dev/null +++ b/src/main/python/plugins/slcp/echo10/Writer.py @@ -0,0 +1,39 @@ +import logging +import os +import os.path + +from edge.writer.solrtemplateresponsewriter import SolrTemplateResponseWriter +from edge.opensearch.solrtemplateresponse import SolrTemplateResponse + +class Writer(SolrTemplateResponseWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = SolrTemplateResponse(self._configuration, searchUrl, searchParams) + response.setTemplate(self.template) + + return response.generate(solrResponse, pretty=pretty) + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + queries = [] + + for key, value in parameters.iteritems(): + if key == 'id': + queries.append('id:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'slcpShortName': + queries.append('SlcpShortName:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'shortName': + queries.append('ShortName:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'nexusShortName': + queries.append('GlobalAttrNexusShortName:' + self._urlEncodeSolrQueryValue(value)) + + query = 'q='+'+AND+'.join(queries)+'&version=2.2&indent=on&wt=json'+'&rows='+str(entriesPerPage) + + logging.debug('solr query: '+query) + + return query http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/echo10/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/echo10/__init__.py b/src/main/python/plugins/slcp/echo10/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/echo10/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/echo10/plugin.conf b/src/main/python/plugins/slcp/echo10/plugin.conf new file mode 100644 index 0000000..b9f3e23 --- /dev/null +++ b/src/main/python/plugins/slcp/echo10/plugin.conf @@ -0,0 +1,8 @@ +[solr] +datasetUrl=http://localhost:8983/solr/dataset +entriesPerPage=1 +parameters=id,slcpShortName,shortName,nexusShortName + +[service] +url=http://localhost:8890 +template=template.xml http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/echo10/template.xml ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/echo10/template.xml b/src/main/python/plugins/slcp/echo10/template.xml new file mode 100644 index 0000000..99de2f8 --- /dev/null +++ b/src/main/python/plugins/slcp/echo10/template.xml @@ -0,0 +1,190 @@ +<Collection> +<ShortName>{{ docs[0]['ShortName'] }}</ShortName> +<VersionId>{{ docs[0]['VersionId'] }}</VersionId> +<LastUpdate>{{ docs[0]['LastUpdate'] }}</LastUpdate> +<LongName>{{ docs[0]['LongName'] }}</LongName> +<DataSetId>{{ docs[0]['DataSetId'] }}</DataSetId> +<Description>{{ docs[0]['Description'] }}</Description> +<ProcessingCenter>{{ docs[0]['ProcessingCenter'] }}</ProcessingCenter> +<ProcessingLevelId>{{ docs[0]['ProcessingLevelId'] }}</ProcessingLevelId> +<ArchiveCenter>{{ docs[0]['ArchiveCenter'] }}</ArchiveCenter> +<DataFormat>{{ docs[0]['DataFormat'] }}</DataFormat> +<SpatialKeywords> +{% for keyword in docs[0]['SpatialKeywords-Keyword'] %} +<Keyword>{{ keyword }}</Keyword> +{% endfor %} +</SpatialKeywords> +<Temporal> +{% for dt in docs[0]['BeginningEndingDateTime'] %} +<RangeDateTime> +{% if ' ' not in dt %} +<BeginningDateTime>{{ dt }}</BeginningDateTime> +<EndingDateTime>{{ dt }}</EndingDateTime> +{% else %} +{% if '*' not in dt[1:dt.index(' ')] %} +<BeginningDateTime>{{ dt[1:dt.index(' ')] }}</BeginningDateTime> +{% else %} +<BeginningDateTime></BeginningDateTime> +{% endif %} +{% if '*' not in dt %} +<EndingDateTime>{{ dt[dt.rindex(' ')+1:-1] }}</EndingDateTime> +{% else %} +<EndingDateTime></EndingDateTime> +{% endif %} +{% endif %} +</RangeDateTime> +{% endfor %} +</Temporal> +{% if docs[0]['CategoryKeyword'] %} +<ScienceKeywords> +{% for i in range(docs[0]['CategoryKeyword']|count) %} +<ScienceKeyword> +<CategoryKeyword>{{ docs[0]['CategoryKeyword'][i] }}</CategoryKeyword> +<TopicKeyword>{{ docs[0]['TopicKeyword'][i] }}</TopicKeyword> +<TermKeyword>{{ docs[0]['TermKeyword'][i] }}</TermKeyword> +<VariableLevel1Keyword> +<Value>{{ docs[0]['VariableLevel1Keyword'][i] }}</Value> +</VariableLevel1Keyword> +{% if docs[0]['DetailedVariableKeyword'] %} +<DetailedVariableKeyword>{{ docs[0]['DetailedVariableKeyword'][i] }}</DetailedVariableKeyword> +{% endif %} +</ScienceKeyword> +{% endfor %} +</ScienceKeywords> +{% endif %} +<Platforms> +{% for i in range(docs[0]['Platform-ShortName']|count) %} +<Platform> +<ShortName>{{ docs[0]['Platform-ShortName'][i] }}</ShortName> +<LongName>{{ docs[0]['Platform-LongName'][i] }}</LongName> +{% if docs[0]['Instrument-ShortName_' + i|string] %} +<Instruments> +{% for j in range(docs[0]['Instrument-ShortName_' + i|string]|count) %} +<Instrument> +<ShortName>{{ docs[0]['Instrument-ShortName_' + i|string][j] }}</ShortName> +<LongName>{{ docs[0]['Instrument-LongName_' + i|string][j] }}</LongName> +{% if docs[0]['Sensor-ShortName_' + i|string + '_' + j|string] %} +<Sensors> +{% for k in range(docs[0]['Sensor-ShortName_' + i|string + '_' + j|string]|count) %} +<Sensor> +<ShortName>{{ docs[0]['Sensor-ShortName_' + i|string + '_' + j|string][k] }}</ShortName> +<LongName>{{ docs[0]['Sensor-LongName_' + i|string + '_' + j|string][k] }}</LongName> +</Sensor> +{% endfor %} +</Sensors> +{% endif %} +</Instrument> +{% endfor %} +</Instruments> +{% endif %} +</Platform> +{% endfor %} +</Platforms> +<AdditionalAttributes> +{% if docs[0]['TemporalResolution'] %} +<AdditionalAttribute> +<Name>Temporal Resolution</Name> +<Value>{{docs[0]['TemporalResolution']}}</Value> +</AdditionalAttribute> +{% endif %} +{% if docs[0]['LatitudeResolution'] %} +<AdditionalAttribute> +<Name>Spatial Resolution</Name> +<Value>{{docs[0]['LatitudeResolution']}} degrees (Latitude) x {{docs[0]['LongitudeResolution']}} degrees (Longitude)</Value> +</AdditionalAttribute> +{% endif %} +{% if docs[0]['AcrossTrackResolution'] %} +<AdditionalAttribute> +<Name>Spatial Resolution</Name> +<Value>{{docs[0]['AlongTrackResolution'] / 1000 }} km (Along) x {{docs[0]['AcrossTrackResolution'] / 1000 }} km (Across)</Value> +</AdditionalAttribute> +{% endif %} +{% if docs[0]['DOI'] %} +<AdditionalAttribute> +<Name>DOI</Name> +<Value>{{docs[0]['DOI']}}</Value> +</AdditionalAttribute> +{% endif %} +{% for key in ['GlobalAttrTitle', 'GlobalAttrDescription', 'GlobalAttrSource', 'GlobalAttrContact', 'GlobalAttrUnits'] %} +{% if docs[0][key] %} +<AdditionalAttribute> +<Name>{{ key }}</Name> +<Value>{{ docs[0][key] }}</Value> +</AdditionalAttribute> +{% endif %} +{% endfor %} +</AdditionalAttributes> +<Campaigns> +{% for i in range(docs[0]['Campaign-ShortName']|count) %} +<Campaign> +<ShortName>{{ docs[0]['Campaign-ShortName'][i] }}</ShortName> +<LongName>{{ docs[0]['Campaign-LongName'][i] }}</LongName> +</Campaign> +{% endfor %} +</Campaigns> +{% if docs[0]['OnlineAccessURL-URL'] %} +<OnlineAccessURLs> +{% for i in range(docs[0]['OnlineAccessURL-URL']|count) %} +<OnlineAccessURL> +<URL>{{ docs[0]['OnlineAccessURL-URL'][i] }}</URL> +<URLDescription>{{ docs[0]['OnlineAccessURL-URLDescription'][i] }}</URLDescription> +</OnlineAccessURL> +{% endfor %} +</OnlineAccessURLs> +{% endif %} +{% if docs[0]['OnlineResource-URL'] %} +<OnlineResources> +{% for i in range(docs[0]['OnlineResource-URL']|count) %} +<OnlineResource> +<URL>{{ docs[0]['OnlineResource-URL'][i] }}</URL> +{% if docs[0]['OnlineResource-Description'] %} +<Description>{{ docs[0]['OnlineResource-Description'][i] }}</Description> +{% endif %} +<Type>{{ docs[0]['OnlineResource-Type'][i] }}</Type> +</OnlineResource> +{% endfor %} +</OnlineResources> +{% endif %} +{% if docs[0]['Spatial-Geometry'] %} +<Spatial> +<HorizontalSpatialDomain> +<Geometry> +{% for box in docs[0]['Spatial-Box'] %} +<BoundingRectangle> +<WestBoundingCoordinate>{{ box.split()[1] }}</WestBoundingCoordinate> +<NorthBoundingCoordinate>{{ box.split()[2] }}</NorthBoundingCoordinate> +<EastBoundingCoordinate>{{ box.split()[3] }}</EastBoundingCoordinate> +<SouthBoundingCoordinate>{{ box.split()[0] }}</SouthBoundingCoordinate> +</BoundingRectangle> +{% endfor %} +{% for polygon in docs[0]['Spatial-Polygon'] %} +<GPolygon> +<Boundary> +{% for i in range(0, polygon.split()|count, 2) %} +<Point> +<PointLongitude>{{ polygon.split()[i+1] }}</PointLongitude> +<PointLatitude>{{ polygon.split()[i] }}</PointLatitude> +</Point> +{% endfor %} +</Boundary> +</GPolygon> +{% endfor %} +</Geometry> +</HorizontalSpatialDomain> +</Spatial> +{% endif %} +<CollectionAssociations> +{% for i in range(docs[0]['CollectionAssociation-ShortName']|count) %} +<CollectionAssociation> +<ShortName>{{ docs[0]['CollectionAssociation-ShortName'][i] }}</ShortName> +<LongName>{{ docs[0]['CollectionAssociation-LongName'][i] }}</LongName> +<VersionId>{{ docs[0]['CollectionAssociation-VersionId'][i] }}</VersionId> +<CollectionType>{{ docs[0]['CollectionAssociation-CollectionType'][i] }}</CollectionType> +<CollectionUse>{{ docs[0]['CollectionAssociation-CollectionUse'][i] }}</CollectionUse> +{% if 'CollectionAssociation-URL' in docs[0] %} +<URL>{{ docs[0]['CollectionAssociation-URL'][i] }}</URL> +{% endif %} +</CollectionAssociation> +{% endfor %} +</CollectionAssociations> +</Collection> http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/facet/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/facet/Writer.py b/src/main/python/plugins/slcp/facet/Writer.py new file mode 100644 index 0000000..4194bde --- /dev/null +++ b/src/main/python/plugins/slcp/facet/Writer.py @@ -0,0 +1,70 @@ +import logging +import os +import os.path +import urllib + +from edge.writer.solrtemplateresponsewriter import SolrTemplateResponseWriter +from edge.response.solrfacettemplateresponse import SolrFacetTemplateResponse + +class Writer(SolrTemplateResponseWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + self.facet = True + self.contentType = 'application/json' + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = SolrFacetTemplateResponse(self.facetDefs) + response.setTemplate(self.template) + + return response.generate(solrResponse, pretty=pretty) + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + queries = [] + filterQueries = [] + + for key, value in parameters.iteritems(): + if value != "": + if key == 'keyword': + queries.append(urllib.quote(value)) + elif key == 'startTime': + queries.append('EndingDateTime-Internal:['+value+'%20TO%20*]') + elif key == 'endTime': + queries.append('BeginningDateTime:[*%20TO%20'+value+']') + elif key == 'bbox': + coordinates = value.split(",") + filterQueries.append('Spatial-Geometry:[' + coordinates[1] + ',' + coordinates[0] + '%20TO%20' + coordinates[3] + ',' + coordinates[2] + ']') + elif key == 'concept_id': + queries.append('concept-id:' + self._urlEncodeSolrQueryValue(value)) + + for key, value in facets.iteritems(): + tagKey = '{!tag=' + key + '}' + key + if type(value) is list: + if (len(value) == 1): + filterQueries.append(tagKey + ':' + self._urlEncodeSolrQueryValue(value[0])) + else: + filterQueries.append(tagKey + ':(' + '+OR+'.join([ self._urlEncodeSolrQueryValue(x) for x in value ]) + ")") + else: + filterQueries.append(tagKey + ':' + self._urlEncodeSolrQueryValue(value)) + + if len(queries) == 0: + queries.append('*:*') + + query = 'q='+'+AND+'.join(queries)+'&version=2.2&indent=on&wt=json' + + if len(filterQueries) > 0: + query += '&fq='+'&fq='.join(filterQueries) + + if self.facet: + query += '&rows=0&facet=true&facet.limit=-1&' + query += '&'.join(['facet.field={!ex=' + facet +'}' + facet if facet in facets else 'facet.field=' + facet for facet in self.facetDefs.values()]) + else: + query += '&start='+str(startIndex)+'&rows='+str(entriesPerPage) + + logging.debug('solr query: '+query) + + return query http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/facet/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/facet/__init__.py b/src/main/python/plugins/slcp/facet/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/facet/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/facet/plugin.conf b/src/main/python/plugins/slcp/facet/plugin.conf new file mode 100644 index 0000000..21627fd --- /dev/null +++ b/src/main/python/plugins/slcp/facet/plugin.conf @@ -0,0 +1,11 @@ +[solr] +datasetUrl=http://localhost:8983/solr/dataset +entriesPerPage=10 +maxEntriesPerPage=2000 +defaultSearchParam=keyword +parameters=keyword,startTime,endTime,bbox,concept_id +facets={"Collection": "Collection", "Processing_Level": "ProcessingLevelBin", "Swath_Spatial_Resolution": "SwathSpatialResolution", "Grid_Spatial_Resolution": "GridSpatialResolution", "Temporal_Resolution": "TemporalResolution", "Parameter": "TermKeyword"} + +[service] +url=http://localhost:8890 +template=template.xml http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/facet/template.xml ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/facet/template.xml b/src/main/python/plugins/slcp/facet/template.xml new file mode 100644 index 0000000..c202f54 --- /dev/null +++ b/src/main/python/plugins/slcp/facet/template.xml @@ -0,0 +1,41 @@ +{ +"facets":[ +{% for key, facet in facetDefs.iteritems() %} +{ +"field":"{{ key }}", +"values":[ +{% if facet == 'GridSpatialResolution' or facet == 'SwathSpatialResolution' %} +{% for i in range(facets['facet_fields'][facet]|count, 0, -2) %} +{ +"count":{{ facets['facet_fields'][facet][i-1] }}, +"value":"{{ facets['facet_fields'][facet][i-2] }}", +"displayedValue":"{{ facets['facet_fields'][facet][i-2] }}{% if facet == 'GridSpatialResolution' %} degree(s){% endif %}{% if facet == 'SwathSpatialResolution' %} km{% endif %}" +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +{% else %} +{% for i in range(0, facets['facet_fields'][facet]|count, 2) %} +{ +"count":{{ facets['facet_fields'][facet][i+1] }}, +"value":"{{ facets['facet_fields'][facet][i] }}", +{% if facet == 'ProcessingLevelBin' %} +"displayedValue":"Level-{{ facets['facet_fields'][facet][i] }}{% if facets['facet_fields'][facet][i] == '2' %} (Swath){% elif facets['facet_fields'][facet][i] == '3' %} (Grid){% elif facets['facet_fields'][facet][i] == '4' %} (Blended){% endif %}" +{% else %} +"displayedValue":"{{ facets['facet_fields'][facet][i] }}" +{% endif %} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +{% endif %} +] +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/granule/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/granule/Writer.py b/src/main/python/plugins/slcp/granule/Writer.py new file mode 100644 index 0000000..e462e1c --- /dev/null +++ b/src/main/python/plugins/slcp/granule/Writer.py @@ -0,0 +1,79 @@ +import logging +import os +import os.path +import urllib +import json + +from edge.writer.solrtemplateresponsewriter import SolrTemplateResponseWriter +from edge.opensearch.solrtemplateresponse import SolrTemplateResponse + +class Writer(SolrTemplateResponseWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = SolrTemplateResponse(self._configuration, searchUrl, searchParams) + response.setTemplate(self.template) + response.variables['serviceUrl'] = self._configuration.get('service', 'url') + + return response.generate(solrResponse, pretty=pretty) + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + sortKeys = json.loads(self._configuration.get('solr', 'sortKeys')) + + queries = [] + filterQueries = [] + sort = None + sortDir = 'asc' + + for key, value in parameters.iteritems(): + if value != "": + if key == 'keyword': + queries.append(urllib.quote(value)) + elif key == 'startTime': + queries.append('EndingDateTime:['+value+'%20TO%20*]') + elif key == 'endTime': + queries.append('BeginningDateTime:[*%20TO%20'+value+']') + elif key == 'bbox': + coordinates = value.split(",") + filterQueries.append('Spatial-Geometry:[' + coordinates[1] + ',' + coordinates[0] + '%20TO%20' + coordinates[3] + ',' + coordinates[2] + ']') + elif key == 'shortName': + queries.append('ShortName:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'sortKey': + if value in sortKeys.keys(): + sort = sortKeys[value] + elif key == 'sortDir': + sortDir = value + + for key, value in facets.iteritems(): + if type(value) is list: + if (len(value) == 1): + filterQueries.append(key + ':' + self._urlEncodeSolrQueryValue(value[0])) + else: + filterQueries.append(key + ':(' + '+OR+'.join([ self._urlEncodeSolrQueryValue(x) for x in value ]) + ")") + else: + filterQueries.append(key + ':' + self._urlEncodeSolrQueryValue(value)) + + if len(queries) == 0: + queries.append('*:*') + + query = 'q='+'+AND+'.join(queries)+'&version=2.2&indent=on&wt=json' + + if len(filterQueries) > 0: + query += '&fq='+'+AND+'.join(filterQueries) + + if self.facet: + query += '&rows=0&facet=true&facet.limit=-1&facet.mincount=1&' + query += '&'.join(['facet.field=' + facet for facet in self.facetDefs.values()]) + else: + query += '&start='+str(startIndex)+'&rows='+str(entriesPerPage) + if sort is not None: + query += '&sort=' + urllib.quote(sort + ' ' + sortDir) + + logging.debug('solr query: '+query) + + return query http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/granule/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/granule/__init__.py b/src/main/python/plugins/slcp/granule/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/granule/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/granule/plugin.conf b/src/main/python/plugins/slcp/granule/plugin.conf new file mode 100644 index 0000000..2095306 --- /dev/null +++ b/src/main/python/plugins/slcp/granule/plugin.conf @@ -0,0 +1,12 @@ +[solr] +datasetUrl=http://localhost:8983/solr/granule +entriesPerPage=10 +maxEntriesPerPage=2000 +defaultSearchParam=keyword +parameters=keyword,startTime,endTime,bbox,sortKey,sortDir,shortName +facets={} +sortKeys={"Relevance": "score", "Start_Date": "BeginningDateTime", "Stop_Date": "EndingDateTime", "Last_Updated": "LastUpdate"} + +[service] +url=http://localhost:8890 +template=template.xml http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/granule/template.xml ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/granule/template.xml b/src/main/python/plugins/slcp/granule/template.xml new file mode 100755 index 0000000..8457aa4 --- /dev/null +++ b/src/main/python/plugins/slcp/granule/template.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="UTF-8"?> +<feed esipdiscovery:version="1.2" xmlns="http://www.w3.org/2005/Atom"; xmlns:dc="http://purl.org/dc/terms/"; xmlns:echo="http://www.echo.nasa.gov/esip"; xmlns:esipdiscovery="http://commons.esipfed.org/ns/discovery/1.2/"; xmlns:georss="http://www.georss.org/georss"; xmlns:gml="http://www.opengis.net/gml"; xmlns:os="http://a9.com/-/spec/opensearch/1.1/"; xmlns:time="http://a9.com/-/opensearch/extensions/time/1.0/";> +<updated>{{ updated }}</updated> +<id></id> +<author> +<name>SLCP</name> +<email></email> +</author> +<title type="text">SLCP granule metadata</title> +<link href="{{ myself }}" hreflang="en-US" rel="self" type="application/atom+xml" /> +{% if last %}<link href="{{ last }}" hreflang="en-US" rel="last" type="application/atom+xml" />{% endif %} +{% if prev %}<link href="{{ prev }}" hreflang="en-US" rel="previous" type="application/atom+xml" />{% endif %} +{% if next %}<link href="{{ next }}" hreflang="en-US" rel="next" type="application/atom+xml" />{% endif %} +{% if first %}<link href="{{ first }}" hreflang="en-US" rel="first" type="application/atom+xml" />{% endif %} +<link href="https://wiki.earthdata.nasa.gov/display/echo/Open+Search+API+release+information"; hreflang="en-US" rel="describedBy" title="Release Notes" type="text/html"/> +<os:totalResults>{{ numFound }}</os:totalResults> +<os:itemsPerPage>{{ itemsPerPage }}</os:itemsPerPage> +<os:startIndex>{{ startIndex }}</os:startIndex> +{% for doc in docs %} +<entry> +{% if 0 == 1 %} +<id>https://api.echo.nasa.gov:443/opensearch/granules.atom?uid=G752080-PODAAC</id> +{% endif %} +<title type="text">{{ doc['GranuleUR'] }}</title> +<updated>{{ doc['LastUpdate'] }}</updated> +{% for i in range(doc['OnlineAccessURL-URL']|count) %} +<link href="{{ doc['OnlineAccessURL-URL'][i] }}" hreflang="en-US" rel="enclosure" {% if doc['OnlineAccessURL-URLDescription'] and doc['OnlineAccessURL-URLDescription'][i] != '' %} title="Data Access {{ doc['OnlineAccessURL-URLDescription'][i] }}" {% endif %} /> +{% endfor %} +{% if 0 == 1 %} +<dc:identifier>G752080-PODAAC</dc:identifier> +{% endif %} +<dc:date>{{ doc['BeginningDateTime'] }}/{{ doc['EndingDateTime'] }}</dc:date> +{% if 0 == 1 %} +<echo:datasetId>GHRSST Level 2P USA NASA MODIS Aqua SST:1</echo:datasetId> +{% endif %} +<echo:granuleSizeMB>{{ doc['FileSize'] / 1048576 }}</echo:granuleSizeMB> +{% if 0 == 1 %} +<echo:originalFormat>ECHO10</echo:originalFormat> +<echo:dataCenter>PODAAC</echo:dataCenter> +<echo:orbitCalSpatialDomain/> +<echo:coordinateSystem>CARTESIAN</echo:coordinateSystem> +{% endif %} +{% if doc['Spatial-Geometry'][0].startswith('ENVELOPE') %} +{% set box = doc['Spatial-Geometry'][0][9:-1].split(',') %} +<georss:box>{{ box[3] }} {{ box[0] }} {{ box[2] }} {{ box[1] }}</georss:box> +{% endif %} +</entry> +{% endfor %} +</feed> http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/indicator/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/indicator/Writer.py b/src/main/python/plugins/slcp/indicator/Writer.py new file mode 100644 index 0000000..2d02f45 --- /dev/null +++ b/src/main/python/plugins/slcp/indicator/Writer.py @@ -0,0 +1,64 @@ +import logging +import urllib +import json + +from edge.writer.proxywriter import ProxyWriter + +class Writer(ProxyWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + def _generateUrl(self, requestHandler): + url = self._configuration.get('solr', 'url') + parameters = {} + parameters['wt'] = 'json' + parameters['omitHeader'] = 'true' + parameters['q'] = '*:*' + try: + parameters['fq'] = 'id:"' + requestHandler.get_argument('id') + '"' + except: + parameters['fl'] = 'id,name,rate,uncertainties,unit,shortenUnit,abbrUnit,updated_at' + try: + if requestHandler.get_argument('latest').lower() == 'true': + parameters['fl'] = 'xLatest,yLatest,unit,abbrUnit,updated_at' + except: + pass + url += '/select?' + urllib.urlencode(parameters) + logging.debug("proxy to url : " + url) + return url + + def onResponse(self, response): + if response.error: + self.requestHandler.set_status(404) + self.requestHandler.write(str(response.error)) + self.requestHandler.finish() + else: + for name, value in response.headers.iteritems(): + logging.debug('header: '+name+':'+value) + self.requestHandler.set_header(name, value) + self.requestHandler.set_header('Access-Control-Allow-Origin', '*') + + solrJson = json.loads(response.body) + if len(solrJson['response']['docs']) > 1: + # Need to order indicators accordingly + solrJsonClone = {} + solrJsonClone['response'] = {} + solrJsonClone['response']['start'] = solrJson['response']['start'] + solrJsonClone['response']['numFound'] = solrJson['response']['numFound'] + solrJsonClone['response']['docs'] = [] + + indicators = {} + for doc in solrJson['response']['docs']: + indicators[doc['id']] = doc + for indicator in self._configuration.get('solr', 'ordering').split(','): + if indicator in indicators: + solrJsonClone['response']['docs'].append(indicators[indicator]) + solrJson = solrJsonClone + for doc in solrJson['response']['docs']: + if 'uncertainties' in doc: + if doc['id'] in self._configuration.get('solr', 'uncertainties').split(','): + doc['uncertainties'] = int(round(doc['uncertainties'])) + doc['rate'] = int(round(doc['rate'])) + + self.requestHandler.write(solrJson) + self.requestHandler.finish() http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/indicator/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/indicator/__init__.py b/src/main/python/plugins/slcp/indicator/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/indicator/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/indicator/plugin.conf b/src/main/python/plugins/slcp/indicator/plugin.conf new file mode 100644 index 0000000..5f6b9a6 --- /dev/null +++ b/src/main/python/plugins/slcp/indicator/plugin.conf @@ -0,0 +1,4 @@ +[solr] +url=http://localhost:8983/solr/indicator +ordering=Global_Mean_Sea_Level,Ocean_Mass,Steric_Height,Greenland_Mass_Variation,Antarctica_Mass_Variation +uncertainties=Greenland_Mass_Variation,Antarctica_Mass_Variation http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/stats/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/stats/Writer.py b/src/main/python/plugins/slcp/stats/Writer.py new file mode 100644 index 0000000..f7675af --- /dev/null +++ b/src/main/python/plugins/slcp/stats/Writer.py @@ -0,0 +1,57 @@ +import logging +import os +import os.path +import urllib + +from edge.writer.solrtemplateresponsewriter import SolrTemplateResponseWriter +from edge.response.solrjsontemplateresponse import SolrJsonTemplateResponse + +class Writer(SolrTemplateResponseWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + self.contentType = 'application/json' + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = SolrJsonTemplateResponse(parameters=searchParams) + response.setTemplate(self.template) + + return response.generate(solrResponse, pretty=pretty) + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + queries = [] + filterQueries = [] + start = '*' + end = '*' + + for key, value in parameters.iteritems(): + if value != "": + if key == 'keyword': + queries.append(urllib.quote(value)) + elif key == 'ds': + filterQueries.append('datasetShortName:%s' % value) + elif key == 'startTime': + start = value + elif key == 'endTime': + end = value + elif key == 'basinId': + filterQueries.append('basinId_i:%s' % value) + elif key == 'basinName': + filterQueries.append('basinName:%s' % value) + filterQueries.append('(time:[' + start + '%20TO%20' + end + '])') + + if len(queries) == 0: + queries.append('*:*') + + query = 'q='+'+AND+'.join(queries)+'&version=2.2&indent=on&wt=json&start='+str(startIndex)+'&rows='+str(entriesPerPage) + + if len(filterQueries) > 0: + query += '&fq='+'+AND+'.join(filterQueries) + + logging.debug('solr query: '+query) + + return query http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/stats/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/stats/__init__.py b/src/main/python/plugins/slcp/stats/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/stats/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/stats/plugin.conf b/src/main/python/plugins/slcp/stats/plugin.conf new file mode 100644 index 0000000..7ea6616 --- /dev/null +++ b/src/main/python/plugins/slcp/stats/plugin.conf @@ -0,0 +1,11 @@ +[solr] +datasetUrl=http://localhost:8983/solr/stats +entriesPerPage=2147483647 +maxEntriesPerPage=2147483647 +defaultSearchParam=keyword +parameters=keyword,ds,basinId,basinName,startTime,endTime +facets={} + +[service] +url=http://localhost:8890 +template=template.json http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/stats/template.json ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/stats/template.json b/src/main/python/plugins/slcp/stats/template.json new file mode 100755 index 0000000..3b21951 --- /dev/null +++ b/src/main/python/plugins/slcp/stats/template.json @@ -0,0 +1,40 @@ +{ + "stats": {}, + "meta": [ + { + "shortName": "{{ parameters['ds'] }}", + "basinId": {{ parameters['basinId'] | jsonify }}, + "time": { + "start": {{ parameters['startTime'] }}, + "stop": {{ parameters['endTime'] }} + } + } + ], + "data": [ + {% for doc in docs %} +[ + { + "std": {{ doc['std'] }}, + "cnt": {{ doc['cnt'] }}, + "minSeasonalLowPass": {{ doc['minSeasonalLowPass'] | jsonify }}, + "minSeasonal": {{ doc['minSeasonal'] | jsonify }}, + "maxLowPass": {{ doc['maxLowPass'] | jsonify }}, + "min": {{ doc['min'] }}, + "max": {{ doc['max'] }}, + "meanSeasonal": {{ doc['meanSeasonal'] | jsonify }}, + "ds": 0, + "meanSeasonalLowPass": {{ doc['meanSeasonalLowPass'] | jsonify }}, + "maxSeasonalLowPass": {{ doc['maxSeasonalLowPass'] | jsonify }}, + "time": {{ doc['time'] }}, + "maxSeasonal": {{ doc['maxSeasonal'] | jsonify }}, + "meanLowPass": {{ doc['meanLowPass'] | jsonify }}, + "minLowPass": {{ doc['minLowPass'] | jsonify }}, + "mean": {{ doc['mean'] }} + } + ] + {% if not loop.last %} +, +{% endif %} +{% endfor %} + ] +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/suggest/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/suggest/Writer.py b/src/main/python/plugins/slcp/suggest/Writer.py new file mode 100644 index 0000000..5fec63d --- /dev/null +++ b/src/main/python/plugins/slcp/suggest/Writer.py @@ -0,0 +1,22 @@ +import logging +import urllib + +from edge.writer.proxywriter import ProxyWriter + +class Writer(ProxyWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + def _generateUrl(self, requestHandler): + url = self._configuration.get('solr', 'url') + parameters = {} + parameters['wt'] = 'json' + parameters['suggest'] = 'true' + #parameters['suggest.build'] = 'true' + try: + parameters['suggest.q'] = requestHandler.get_argument('keyword') + url += '/suggest?' + urllib.urlencode(parameters) + except: + raise Exception('Missing keyword parameter.') + logging.debug("proxy to url : " + url) + return url http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/suggest/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/suggest/__init__.py b/src/main/python/plugins/slcp/suggest/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/suggest/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/suggest/plugin.conf b/src/main/python/plugins/slcp/suggest/plugin.conf new file mode 100644 index 0000000..55aa1a3 --- /dev/null +++ b/src/main/python/plugins/slcp/suggest/plugin.conf @@ -0,0 +1,2 @@ +[solr] +url=http://localhost:8983/solr/dataset http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/umm-json/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/umm-json/Writer.py b/src/main/python/plugins/slcp/umm-json/Writer.py new file mode 100644 index 0000000..388e610 --- /dev/null +++ b/src/main/python/plugins/slcp/umm-json/Writer.py @@ -0,0 +1,39 @@ +import logging +import os +import os.path + +from edge.writer.solrtemplateresponsewriter import SolrTemplateResponseWriter +from edge.response.solrjsontemplateresponse import SolrJsonTemplateResponse + +class Writer(SolrTemplateResponseWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + self.contentType = 'application/json' + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = SolrJsonTemplateResponse() + response.setTemplate(self.template) + + return response.generate(solrResponse, pretty=pretty) + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + queries = [] + + for key, value in parameters.iteritems(): + if key == 'id': + queries.append('id:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'slcpShortName': + queries.append('SlcpShortName:' + self._urlEncodeSolrQueryValue(value)) + elif key == 'shortName': + queries.append('ShortName:' + self._urlEncodeSolrQueryValue(value)) + + query = 'q='+'+AND+'.join(queries)+'&version=2.2&indent=on&wt=json'+'&rows='+str(entriesPerPage) + + logging.debug('solr query: '+query) + + return query http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/umm-json/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/umm-json/__init__.py b/src/main/python/plugins/slcp/umm-json/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/umm-json/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/umm-json/plugin.conf b/src/main/python/plugins/slcp/umm-json/plugin.conf new file mode 100644 index 0000000..3836fac --- /dev/null +++ b/src/main/python/plugins/slcp/umm-json/plugin.conf @@ -0,0 +1,8 @@ +[solr] +datasetUrl=http://localhost:8983/solr/dataset +entriesPerPage=1 +parameters=id,slcpShortName,shortName + +[service] +url=http://localhost:8890 +template=template.json http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/slcp/umm-json/template.json ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/slcp/umm-json/template.json b/src/main/python/plugins/slcp/umm-json/template.json new file mode 100644 index 0000000..1e00e8d --- /dev/null +++ b/src/main/python/plugins/slcp/umm-json/template.json @@ -0,0 +1,274 @@ +{ +"Projects": [ +{% for i in range(docs[0]['Campaign-ShortName']|count) %} +{ +"ShortName" : {{ docs[0]['Campaign-ShortName'][i] | jsonify }}, +"LongName" : {{ docs[0]['Campaign-LongName'][i] | jsonify }} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +], +{% if docs[0]['Spatial-Geometry'] %} +"SpatialExtent" : { +"SpatialCoverageType" : "HORIZONTAL", +"HorizontalSpatialDomain" : { +"Geometry" : { +"CoordinateSystem" : {{ docs[0]['CoordinateSystem'] | jsonify }}, +"BoundingRectangles" : [ +{% for box in docs[0]['Spatial-Box'] %} +{ +"WestBoundingCoordinate" : {{ box.split()[1] }}, +"NorthBoundingCoordinate" : {{ box.split()[2] }}, +"EastBoundingCoordinate" : {{ box.split()[3] }}, +"SouthBoundingCoordinate" : {{ box.split()[0] }} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +} +}, +"GranuleSpatialRepresentation" : {{ docs[0]['CoordinateSystem'] | jsonify }} +}, +{% endif %} +"Distributions" : [ { +"DistributionFormat" : {{ docs[0]['DataFormat'] | jsonify }} +} ], +{% if docs[0]['CategoryKeyword'] %} +"ScienceKeywords" : [ +{% for i in range(docs[0]['CategoryKeyword']|count) %} +{ +"Category" : {{ docs[0]['CategoryKeyword'][i] | jsonify }}, +"Topic" : {{ docs[0]['TopicKeyword'][i] | jsonify }}, +"Term" : {{ docs[0]['TermKeyword'][i] | jsonify }}, +"VariableLevel1" : {{ docs[0]['VariableLevel1Keyword'][i] | jsonify }} +{% if docs[0]['DetailedVariableKeyword'] %} +,"DetailedVariable" : {{ docs[0]['DetailedVariableKeyword'][i] | jsonify }} +{% endif %} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +], +{% endif %} +"TemporalExtents" : [ { + "RangeDateTimes" : [ +{% for dt in docs[0]['BeginningEndingDateTime'] %} +{ +{% if ' ' not in dt %} +"BeginningDateTime" : "{{ dt }}", +"EndingDateTime" : "{{ dt }}" +{% else %} +"BeginningDateTime" : "{{ dt[1:dt.index(' ')] }}" +{% if '*' not in dt %} +, +"EndingDateTime" : "{{ dt[dt.rindex(' ')+1:-1] }}" +{% endif %} +{% endif %} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +} ], +"ProcessingLevel" : { +"Id" : {{ docs[0]['ProcessingLevelId'] | jsonify }} +}, +"ShortName" : {{ docs[0]['ShortName'] | jsonify }}, +"EntryTitle" : {{ docs[0]['LongName'] | jsonify }}, +"RelatedUrls" : [ +{% for i in range(docs[0]['OnlineAccessURL-URL']|count) %} +{ +"Description" : {{ docs[0]['OnlineAccessURL-URLDescription'][i] | jsonify }}, +"Relation" : [ "GET DATA" ], +"URLs" : [ {{ docs[0]['OnlineAccessURL-URL'][i] | jsonify }} ] +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +{% if docs[0]['OnlineResource-URL'] %} +, +{% endif %} +{% for i in range(docs[0]['OnlineResource-URL']|count) %} +{ +{% if docs[0]['OnlineResource-Description'][i] != '' %} +"Description" : {{ docs[0]['OnlineResource-Description'][i] | jsonify }}, +{% endif %} +"URLs" : [ {{ docs[0]['OnlineResource-URL'][i] | jsonify }} ] +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +], +"DataDates" : [ { +"Date" : "{{ docs[0]['LastUpdate'] }}", +"Type" : "UPDATE" +} ], +"Abstract" : {{ docs[0]['Description'] | jsonify }}, +"Version" : {{ docs[0]['VersionId'] | jsonify }}, +"LocationKeywords" : [ +{% for keyword in docs[0]['SpatialKeywords-Keyword'] %} +{ +"Category": "GEOGRAPHIC REGION", + "Type": {{ keyword | jsonify }} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +], +"Platforms" : [ +{% for i in range(docs[0]['Platform-ShortName']|count) %} +{ +"Type" : "SPACECRAFT", +"ShortName" : {{ docs[0]['Platform-ShortName'][i] | jsonify }}, +"LongName" : {{ docs[0]['Platform-LongName'][i] | jsonify }} +{% if docs[0]['Instrument-ShortName_' + i|string] %} +, +"Instruments" : [ +{% for j in range(docs[0]['Instrument-ShortName_' + i|string]|count) %} +{ +"ShortName" : {{ docs[0]['Instrument-ShortName_' + i|string][j] | jsonify }}, +"LongName" : {{ docs[0]['Instrument-LongName_' + i|string][j] | jsonify }} +{% if docs[0]['Sensor-ShortName_' + i|string + '_' + j|string] %} +, +"Sensors" : [ +{% for k in range(docs[0]['Sensor-ShortName_' + i|string + '_' + j|string]|count) %} +{ +"ShortName" : {{ docs[0]['Sensor-ShortName_' + i|string + '_' + j|string][k] | jsonify }}, +"LongName" : {{ docs[0]['Sensor-LongName_' + i|string + '_' + j|string][k] | jsonify }} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +{% endif %} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +{% endif %} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +{% if docs[0]['DOI'] %} +, +"DOI" : {"DOI" : {{ docs[0]['DOI'] | jsonify }}} +{% endif %} +, +"AncillaryKeywords" : [ +{% for keyword in docs[0]['Keyword'] %} +{{ keyword | jsonify }} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +], +"DataCenters": [ +{% if docs[0]['ProcessingCenter'] %} +{ +"Roles": ["PROCESSOR"], +"ShortName": {{ docs[0]['ProcessingCenter'] | jsonify }} +} +{% endif %} +{% if docs[0]['ArchiveCenter'] %} +{% if docs[0]['ProcessingCenter'] %} +, +{% endif %} +{ +"Roles": ["ARCHIVER"], +"ShortName": {{ docs[0]['ArchiveCenter'] | jsonify }} +} +{% endif %} +], +{% if docs[0]['LatitudeResolution'] %} +"GeographicCoordinateSystemType" : { +"GeographicCoordinateUnits" : "degrees", +"LatitudeResolution" : {{ docs[0]['LatitudeResolution'] }}, +"LongitudeResolution" : {{ docs[0]['LongitudeResolution'] }} +}, +{% endif %} +{% if docs[0]['TemporalResolution'] %} +"TemporalKeywords" : [ +{{ docs[0]['TemporalResolution'] | jsonify }} +], +{% endif %} +"MetadataAssociations":[ +{% for i in range(docs[0]['CollectionAssociation-ShortName']|count) %} +{ +"EntryId": {{ docs[0]['CollectionAssociation-ShortName'][i] | jsonify }}, +"Description": {{ docs[0]['CollectionAssociation-CollectionUse'][i] | jsonify }}, +"Type": {{ docs[0]['CollectionAssociation-CollectionType'][i] | jsonify }}, +"Version": {{ docs[0]['CollectionAssociation-VersionId'][i] | jsonify }}, +"EntryTitle": {{ docs[0]['CollectionAssociation-LongName'][i] | jsonify }} +{% if 'CollectionAssociation-URL' in docs[0] %} +,"URL": {{ docs[0]['CollectionAssociation-URL'][i] | jsonify }} +{% endif %} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +], +"PublicationReferences":[ +{% set publicationKeys = {"PublicationReferences-Title": "Title", "PublicationReferences-Publisher": "Publisher", "PublicationReferences-Author": "Author", "PublicationReferences-PublicationDate": "PublicationDate", "PublicationReferences-Series": "Series", "PublicationReferences-Edition": "Edition", "PublicationReferences-Volume": "Volume", "PublicationReferences-Issue": "Issue", "PublicationReferences-ReportNumber": "ReportNumber", "PublicationReferences-PublicationPlace": "PublicationPlace", "PublicationReferences-Pages": "Pages", "PublicationReferences-ISBN": "ISBN", "PublicationReferences-OtherReferenceDetails": "OtherReferenceDetails"} %} +{% for i in range(docs[0]['PublicationReferences-Title']|count) %} +{ +{% for key, value in publicationKeys.iteritems() %} +"{{ value }}": {{ docs[0][key][i].strip() | jsonify }}, +{% endfor %} +"DOI": {"DOI" : {{ docs[0]['PublicationReferences-DOI'][i] | jsonify}} } +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +], +"AdditionalAttributes" : [ +{% set hasAttrs = False %} +{% if docs[0]['NexusDataBeginningDateTime'] %} +{% set hasAttrs = True %} +{"Name": "NexusDataBeginningDateTime", "Description": "Beginning date of data available in Nexus", "DataType": "DATETIME", "Value": "{{ docs[0]['NexusDataBeginningDateTime'] }}"} +{% endif %} +{% if docs[0]['NexusDataEndingDateTime'] %} +{% if hasAttrs %},{% endif %} +{% set hasAttrs = True %} +{"Name": "NexusDataEndingDateTime", "Description": "Ending date of data available in Nexus", "DataType": "DATETIME", "Value": "{{ docs[0]['NexusDataEndingDateTime'] }}"} +{% endif %} +{% if docs[0]['NexusShortName'] %} +{% if hasAttrs %},{% endif %} +{% set hasAttrs = True %} +{"Name": "NexusShortName", "Description": "Name of dataset in Nexus", "DataType": "STRING", "Value": "{{ docs[0]['NexusShortName'][0] }}"} +{% endif %} +{% if docs[0]['LatitudeResolution'] %} +{% if hasAttrs %},{% endif %} +{% set hasAttrs = True %} +{"Name": "Spatial Resolution", "Description": "Spatial resolution of dataset", "DataType": "STRING", "Value": "{{docs[0]['LatitudeResolution']}} degrees (Latitude) x {{docs[0]['LongitudeResolution']}} degrees (Longitude)"} +{% endif %} +{% if docs[0]['AcrossTrackResolution'] %} +{% if hasAttrs %},{% endif %} +{% set hasAttrs = True %} +{"Name": "Spatial Resolution", "Description": "Spatial resolution of dataset", "DataType": "STRING", "Value": "{{docs[0]['AlongTrackResolution'] / 1000 }} km (Along) x {{docs[0]['AcrossTrackResolution'] / 1000 }} km (Across)"} +{% endif %} +{% for key in ['GlobalAttrTitle', 'GlobalAttrDescription', 'GlobalAttrSource', 'GlobalAttrContact', 'GlobalAttrUnits'] %} +{% if docs[0][key] %} +{% if hasAttrs %},{% endif %} +{% set hasAttrs = True %} +{"Name": "{{ key }}", "DataType": "STRING", "Value": {{ docs[0][key] | jsonify }}} +{% endif %} +{% endfor %} +] +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs/__init__.py b/src/main/python/plugins/spurs/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs/json/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs/json/Writer.py b/src/main/python/plugins/spurs/json/Writer.py new file mode 100644 index 0000000..126089f --- /dev/null +++ b/src/main/python/plugins/spurs/json/Writer.py @@ -0,0 +1,76 @@ +import logging +import os +import os.path +import urllib +import json + +from edge.writer.solrtemplateresponsewriter import SolrTemplateResponseWriter +from edge.response.solrjsontemplateresponse import SolrJsonTemplateResponse + +class Writer(SolrTemplateResponseWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + self.contentType = 'application/json' + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = SolrJsonTemplateResponse(searchUrl, searchParams) + response.setTemplate(self.template) + + return response.generate(solrResponse, pretty=pretty) + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + variable = json.loads(self._configuration.get('solr', 'variable')) + + queries = [] + filterQueries = [] + sort = None + + for key, value in parameters.iteritems(): + if value != "": + if key == 'keyword': + queries.append(urllib.quote(value)) + elif key == 'startTime': + filterQueries.append('time:['+value+'%20TO%20*]') + elif key == 'endTime': + filterQueries.append('time:[*%20TO%20'+value+']') + elif key == 'bbox': + coordinates = value.split(",") + filterQueries.append('point_srpt:[' + coordinates[1] + ',' + coordinates[0] + '%20TO%20' + coordinates[3] + ',' + coordinates[2] + ']') + elif key == 'variable': + if value.lower() in variable: + filterQueries.append("(" + "+OR+".join([x + ":[*%20TO%20*]" for x in variable[value.lower()]]) + ")") + elif key == 'minDepth': + filterQueries.append('depth:['+value+'%20TO%20*]') + elif key == 'maxDepth': + filterQueries.append('depth:[*%20TO%20'+value+']') + elif key == 'platform': + if type(value) is list: + filterQueries.append('platform:(' + '+OR+'.join(value) + ')') + else: + filterQueries.append('platform:'+value) + + if len(queries) == 0: + queries.append('*:*') + + query = 'q='+'+AND+'.join(queries)+'&wt=json&start='+str(startIndex)+'&rows='+str(entriesPerPage) + + if len(filterQueries) > 0: + query += '&fq='+'+AND+'.join(filterQueries) + + if sort is not None: + query += '&sort=' + sort + + if 'stats' in parameters and parameters['stats'].lower() == 'true': + query += '&stats=true&stats.field={!min=true%20max=true}depth' + + if 'facet' in parameters and parameters['facet'].lower() == 'true': + query += '&facet=true&facet.field=platform&facet.field=device&facet.limit=-1&facet.mincount=1' + + logging.debug('solr query: '+query) + + return query http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs/json/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs/json/__init__.py b/src/main/python/plugins/spurs/json/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs/json/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs/json/plugin.conf b/src/main/python/plugins/spurs/json/plugin.conf new file mode 100644 index 0000000..76c39c8 --- /dev/null +++ b/src/main/python/plugins/spurs/json/plugin.conf @@ -0,0 +1,12 @@ +[solr] +datasetUrl=http://localhost:8983/solr/spurs +entriesPerPage=10 +maxEntriesPerPage=100000 +defaultSearchParam=keyword +parameters=keyword,startTime,endTime,bbox,minDepth,maxDepth,variable,stats,platform,facet +facets={} +variable={"sss" : ["salinity", "salinity1", "salinity_ctd"], "sst" : ["temperature", "temperature1", "TEMP", "temperature_ctd"], "wind" : ["RELWSPD"]} + +[service] +url=http://localhost:8890 +template=template.json http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs/json/template.json ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs/json/template.json b/src/main/python/plugins/spurs/json/template.json new file mode 100755 index 0000000..c445f12 --- /dev/null +++ b/src/main/python/plugins/spurs/json/template.json @@ -0,0 +1,63 @@ +{ +{% if last %}"last": "{{ last }}",{% endif %} +{% if prev %}"prev": "{{ prev }}",{% endif %} +{% if next %}"next": "{{ next }}",{% endif %} +{% if first %}"first": "{{ first }}",{% endif %} +"results":[ +{% for doc in docs %} +{ +"id": "{{ doc['id'] }}", +"time": "{{ doc['time'] }}", +"point": "Point({{ doc['point_srpt'] }})", +"sea_water_temperature": {% if 'temperature' in doc %}{{ doc['temperature'] | jsonify }}{% elif 'temperature1' in doc %}{{ doc['temperature1'] | jsonify }}{% elif 'TEMP' in doc %}{{ doc['TEMP'] | jsonify }}{% elif 'temperature_ctd' in doc %}{{ doc['temperature_ctd'] | jsonify }}{% else %}null{% endif %}, +"sea_water_temperature_depth": {% if 'temperature' in doc or 'temperature1' in doc or 'TEMP' in doc or 'temperature_ctd' in doc %}{{ doc['depth'] | jsonify }}{% else %}null{% endif %}, +"sea_water_temperature_quality": {% if 'temperature' in doc or 'temperature1' in doc or 'TEMP' in doc or 'temperature_ctd' in doc %}0{% else %}null{% endif %}, +"wind_speed": {{ doc['RELWSPD'] | jsonify }}, +"eastward_wind": {% if 'eastward_wind' in doc %}{{ doc['eastward_wind'][0] | jsonify }}{% else %}null{% endif %}, +"northward_wind": {% if 'northward_wind' in doc %}{{ doc['northward_wind'][0] | jsonify }}{% else %}null{% endif %}, +"wind_depth": {% if 'RELWSPD' in doc %}{{ doc['depth'] | jsonify }}{% else %}null{% endif %}, +"wind_quality": {% if 'RELWSPD' in doc %}0{% else %}null{% endif %}, +"sea_water_salinity": {% if 'salinity' in doc %}{{ doc['salinity'] | jsonify }}{% elif 'salinity1' in doc %}{{ doc['salinity1'] | jsonify }}{% elif 'salinity_ctd' in doc %}{{ doc['salinity_ctd'] | jsonify }}{% else %}null{% endif %}, +"sea_water_salinity_depth": {% if 'salinity' in doc or 'salinity1' in doc or 'salinity_ctd' in doc %}{{ doc['depth'] | jsonify }}{% else %}null{% endif %}, +"sea_water_salinity_quality": {% if 'salinity' in doc or 'salinity1' in doc or 'salinity_ctd' in doc %}0{% else %}null{% endif %}, +"platform": {{ doc['platform'] | jsonify }}, +"device": {{ doc['device'] | jsonify }}, +"fileurl": {{ doc['fileurl'] | jsonify }} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +,"totalResults":{{ numFound }} +,"startIndex":{{ startIndex }} +,"itemsPerPage":{{ itemsPerPage }} +{% if stats %} +, +"stats_fields": {{ stats['stats_fields'] | jsonify }} +{% endif %} +{% if facets %} +, +"facets":[ +{% for key, facet in facets['facet_fields'].iteritems() %} +{ +"field": "{{ key }}", +"values":[ +{% for i in range(0, facet|count, 2) %} +{ +"count":{{facet[i+1] }}, +"value": "{{ facet[i] }}" +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +{% endif %} +} http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs2/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs2/__init__.py b/src/main/python/plugins/spurs2/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs2/json/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs2/json/Writer.py b/src/main/python/plugins/spurs2/json/Writer.py new file mode 100644 index 0000000..126089f --- /dev/null +++ b/src/main/python/plugins/spurs2/json/Writer.py @@ -0,0 +1,76 @@ +import logging +import os +import os.path +import urllib +import json + +from edge.writer.solrtemplateresponsewriter import SolrTemplateResponseWriter +from edge.response.solrjsontemplateresponse import SolrJsonTemplateResponse + +class Writer(SolrTemplateResponseWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + self.contentType = 'application/json' + + templatePath = os.path.dirname(configFilePath) + os.sep + templatePath += self._configuration.get('service', 'template') + self.template = self._readTemplate(templatePath) + + def _generateOpenSearchResponse(self, solrResponse, searchText, searchUrl, searchParams, pretty): + response = SolrJsonTemplateResponse(searchUrl, searchParams) + response.setTemplate(self.template) + + return response.generate(solrResponse, pretty=pretty) + + def _constructSolrQuery(self, startIndex, entriesPerPage, parameters, facets): + variable = json.loads(self._configuration.get('solr', 'variable')) + + queries = [] + filterQueries = [] + sort = None + + for key, value in parameters.iteritems(): + if value != "": + if key == 'keyword': + queries.append(urllib.quote(value)) + elif key == 'startTime': + filterQueries.append('time:['+value+'%20TO%20*]') + elif key == 'endTime': + filterQueries.append('time:[*%20TO%20'+value+']') + elif key == 'bbox': + coordinates = value.split(",") + filterQueries.append('point_srpt:[' + coordinates[1] + ',' + coordinates[0] + '%20TO%20' + coordinates[3] + ',' + coordinates[2] + ']') + elif key == 'variable': + if value.lower() in variable: + filterQueries.append("(" + "+OR+".join([x + ":[*%20TO%20*]" for x in variable[value.lower()]]) + ")") + elif key == 'minDepth': + filterQueries.append('depth:['+value+'%20TO%20*]') + elif key == 'maxDepth': + filterQueries.append('depth:[*%20TO%20'+value+']') + elif key == 'platform': + if type(value) is list: + filterQueries.append('platform:(' + '+OR+'.join(value) + ')') + else: + filterQueries.append('platform:'+value) + + if len(queries) == 0: + queries.append('*:*') + + query = 'q='+'+AND+'.join(queries)+'&wt=json&start='+str(startIndex)+'&rows='+str(entriesPerPage) + + if len(filterQueries) > 0: + query += '&fq='+'+AND+'.join(filterQueries) + + if sort is not None: + query += '&sort=' + sort + + if 'stats' in parameters and parameters['stats'].lower() == 'true': + query += '&stats=true&stats.field={!min=true%20max=true}depth' + + if 'facet' in parameters and parameters['facet'].lower() == 'true': + query += '&facet=true&facet.field=platform&facet.field=device&facet.limit=-1&facet.mincount=1' + + logging.debug('solr query: '+query) + + return query http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs2/json/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs2/json/__init__.py b/src/main/python/plugins/spurs2/json/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs2/json/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs2/json/plugin.conf b/src/main/python/plugins/spurs2/json/plugin.conf new file mode 100644 index 0000000..37313d1 --- /dev/null +++ b/src/main/python/plugins/spurs2/json/plugin.conf @@ -0,0 +1,12 @@ +[solr] +datasetUrl=http://localhost:8983/solr/spurs2 +entriesPerPage=10 +maxEntriesPerPage=100000 +defaultSearchParam=keyword +parameters=keyword,startTime,endTime,bbox,minDepth,maxDepth,variable,stats,platform,facet +facets={} +variable={"sss" : ["salinity", "salinity1", "salinity_ctd", "S_41"], "sst" : ["temperature", "temperature1", "TEMP", "temperature_ctd", "T_20"], "wind" : ["RELWSPD"]} + +[service] +url=http://localhost:8890 +template=template.json http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/spurs2/json/template.json ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/spurs2/json/template.json b/src/main/python/plugins/spurs2/json/template.json new file mode 100755 index 0000000..a629a5d --- /dev/null +++ b/src/main/python/plugins/spurs2/json/template.json @@ -0,0 +1,64 @@ +{ +{% if last %}"last": "{{ last }}",{% endif %} +{% if prev %}"prev": "{{ prev }}",{% endif %} +{% if next %}"next": "{{ next }}",{% endif %} +{% if first %}"first": "{{ first }}",{% endif %} +"results":[ +{% for doc in docs %} +{ +"id": "{{ doc['id'] }}", +"time": "{{ doc['time'] }}", +"point": "Point({{ doc['point_srpt'] }})", +"sea_water_temperature": {% if 'temperature' in doc %}{{ doc['temperature'] | jsonify }}{% elif 'temperature1' in doc %}{{ doc['temperature1'] | jsonify }}{% elif 'TEMP' in doc %}{{ doc['TEMP'] | jsonify }}{% elif 'temperature_ctd' in doc %}{{ doc['temperature_ctd'] | jsonify }}{% elif 'T_20' in doc %}{{ doc['T_20'] | jsonify }}{% else %}null{% endif %}, +"sea_water_temperature_depth": {% if 'temperature' in doc or 'temperature1' in doc or 'TEMP' in doc or 'temperature_ctd' in doc or 'T_20' in doc %}{{ doc['depth'] | jsonify }}{% else %}null{% endif %}, +"sea_water_temperature_quality": {% if 'temperature' in doc or 'temperature1' in doc or 'TEMP' in doc or 'temperature_ctd' in doc or 'T_20' in doc %}0{% else %}null{% endif %}, +"wind_speed": {{ doc['RELWSPD'] | jsonify }}, +"eastward_wind": {{ doc['wind_u'] | jsonify }}, +"northward_wind": {{ doc['wind_v'] | jsonify }}, +"wind_depth": {% if 'RELWSPD' in doc %}{{ doc['depth'] | jsonify }}{% else %}null{% endif %}, +"wind_quality": {% if 'RELWSPD' in doc %}0{% else %}null{% endif %}, +"sea_water_salinity": {% if 'salinity' in doc %}{{ doc['salinity'] | jsonify }}{% elif 'salinity1' in doc %}{{ doc['salinity1'] | jsonify }}{% elif 'salinity_ctd' in doc %}{{ doc['salinity_ctd'] | jsonify }}{% elif 'S_41' in doc %}{{ doc['S_41'] | jsonify }}{% else %}null{% endif %}, +"sea_water_salinity_depth": {% if 'salinity' in doc or 'salinity1' in doc or 'salinity_ctd' in doc or 'S_41' in doc %}{{ doc['depth'] | jsonify }}{% else %}null{% endif %}, +"sea_water_salinity_quality": {% if 'salinity' in doc or 'salinity1' in doc or 'salinity_ctd' in doc or 'S_41' in doc %}0{% else %}null{% endif %}, +"platform": {{ doc['platform'] | jsonify }}, +"device": {{ doc['device'] | jsonify }}, +"fileurl": {{ doc['fileurl'] | jsonify }} +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +,"totalResults":{{ numFound }} +,"startIndex":{{ startIndex }} +,"itemsPerPage":{{ itemsPerPage }} +{% if stats %} +, +"stats_fields": {{ stats['stats_fields'] | jsonify }} +{% endif %} +{% if facets %} +, +"facets":[ +{% for key, facet in facets['facet_fields'].iteritems() %} +{ +"field": "{{ key }}", +"values":[ +{% for i in range(0, facet|count, 2) %} +{ +"count":{{facet[i+1] }}, +"value": "{{ facet[i] }}" +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +} +{% if not loop.last %} +, +{% endif %} +{% endfor %} +] +{% endif %} +} + http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/tie/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/tie/__init__.py b/src/main/python/plugins/tie/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/tie/collection/Writer.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/tie/collection/Writer.py b/src/main/python/plugins/tie/collection/Writer.py new file mode 100644 index 0000000..bad8827 --- /dev/null +++ b/src/main/python/plugins/tie/collection/Writer.py @@ -0,0 +1,54 @@ +import logging +import urllib + +from edge.dateutility import DateUtility +from edge.writer.proxywriter import ProxyWriter + +class Writer(ProxyWriter): + def __init__(self, configFilePath): + super(Writer, self).__init__(configFilePath) + + def _generateUrl(self, requestHandler): + url = self._configuration.get('solr', 'url') + parameters = {} + parameters['wt'] = 'json' + parameters['group'] = 'true' + parameters['group.limit'] = -1 + #parameters['facet.limit'] = 10 + parameters['fl'] = 'time,productTypePrefix,productType' + parameters['group.field'] = 'crid' + parameters['omitHeader'] = 'true' + parameters['q'] = '*:*' + parameters['fq'] = [] + parameters['sort'] = 'crid desc' + try: + parameters['fq'].append('collection:"' + requestHandler.get_argument('collection') + '"') + except: + pass + try: + parameters['fq'].append('productType:"' + requestHandler.get_argument('productType') + '"') + except: + pass + try: + start = requestHandler.get_argument('start') + if len(start) == 10: + start += 'T00:00:00' + except: + raise Exception('Missing start parameter.') + try: + end = requestHandler.get_argument('end') + if len(end) == 10: + end += 'T23:59:59' + except: + end = start[0:10] + 'T23:59:59' + + logging.debug('start: ' + start) + logging.debug('end: ' + end) + + start = DateUtility.convertISOToUTCTimestamp(start) + end = DateUtility.convertISOToUTCTimestamp(end) + 999 + parameters['fq'].append('time:[' + str(start) + ' TO ' + str(end) + ']') + + url += '/select?' + urllib.urlencode(parameters, True) + logging.debug("proxy to url : " + url) + return url http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/tie/collection/__init__.py ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/tie/collection/__init__.py b/src/main/python/plugins/tie/collection/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/plugins/tie/collection/plugin.conf ---------------------------------------------------------------------- diff --git a/src/main/python/plugins/tie/collection/plugin.conf b/src/main/python/plugins/tie/collection/plugin.conf new file mode 100644 index 0000000..d82d682 --- /dev/null +++ b/src/main/python/plugins/tie/collection/plugin.conf @@ -0,0 +1,2 @@ +[solr] +url=http://localhost:8983/solr/layer http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/requestresponder.py ---------------------------------------------------------------------- diff --git a/src/main/python/requestresponder.py b/src/main/python/requestresponder.py new file mode 100644 index 0000000..5d27c48 --- /dev/null +++ b/src/main/python/requestresponder.py @@ -0,0 +1,24 @@ +import logging +import ConfigParser + +class RequestResponder(object): + def __init__(self, configFilePath): + #logging.debug('config: '+configFilePath) + self._configuration = ConfigParser.RawConfigParser() + self._configuration.read(configFilePath) + self.requestHandler = None + + def get(self, requestHandler): + self.requestHandler = requestHandler + + def post(self, requestHandler): + self.requestHandler = requestHandler + + def put(self, requestHandler): + self.requestHandler = requestHandler + + def delete(self, requestHandler): + self.requestHandler = requestHandler + + def options(self, requestHandler): + self.requestHandler = requestHandler http://git-wip-us.apache.org/repos/asf/incubator-sdap-edge/blob/53351bf3/src/main/python/requirements.txt ---------------------------------------------------------------------- diff --git a/src/main/python/requirements.txt b/src/main/python/requirements.txt new file mode 100644 index 0000000..e8805c9 --- /dev/null +++ b/src/main/python/requirements.txt @@ -0,0 +1,3 @@ +tornado==4.2.1 +jinja2==2.8 +python-dateutil==2.4.1
