Repository: climate Updated Branches: refs/heads/master 9deb819f7 -> fba1ce638
Adding PO.DAAC data source along with the tests Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/116d16f0 Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/116d16f0 Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/116d16f0 Branch: refs/heads/master Commit: 116d16f039f2f3e3f1c4cfddec447a49afc459aa Parents: 91ec7f3 Author: Omkar20895 <omkarreddy2...@gmail.com> Authored: Tue Aug 9 01:06:40 2016 +0530 Committer: Omkar20895 <omkarreddy2...@gmail.com> Committed: Tue Aug 9 01:06:40 2016 +0530 ---------------------------------------------------------------------- ocw/data_source/podaac.py | 110 +++++++++++++++++++++++++++++++++++++++++ ocw/tests/test_podaac.py | 62 +++++++++++++++++++++++ 2 files changed, 172 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/116d16f0/ocw/data_source/podaac.py ---------------------------------------------------------------------- diff --git a/ocw/data_source/podaac.py b/ocw/data_source/podaac.py new file mode 100644 index 0000000..1eb0195 --- /dev/null +++ b/ocw/data_source/podaac.py @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +sys.path.append('/Users/omkar/Documents/Git/podaacpy/podaac') +from podaac_data_source import Podaac +import numpy as np +from ocw.dataset import Dataset +from netCDF4 import Dataset as netcdf_dataset +from netcdftime import utime +import os, urllib +import xml.etree.ElementTree as ET + + +def _convert_times_to_datetime(time): + '''Convert the time object's values to datetime objects + + The time values are stored as some unit since an epoch. These need to be + converted into datetime objects for the OCW Dataset object. + + :param time: The time object's values to convert + :type time: pydap.model.BaseType + + :returns: list of converted time values as datetime objects + ''' + units = time.units + # parse the time units string into a useful object. + # NOTE: This assumes a 'standard' calendar. It's possible (likely?) that + # users will want to customize this in the future. + parsed_time = utime(units) + return [parsed_time.num2date(x) for x in time[:]] + + + +def load_dataset(variable ,datasetId='', datasetShortName='', name=''): + '''Loads a Dataset from PODAAC + + :param variable: The name of the variable to read from the dataset. + :type variable: :mod:`string` + + :param datasetId: dataset persistent ID. datasetId or \ + shortName is required for a granule search. Example: \ + PODAAC-ASOP2-25X01 + :type datasetId: :mod:`string` + + :param shortName: the shorter name for a dataset. \ + Either shortName or datasetId is required for a \ + granule search. Example: ASCATA-L2-25km + :type shortName: :mod:`string` + + :param name: (Optional) A name for the loaded dataset. + :type name: :mod:`string` + + :returns: A :class:`dataset.Dataset` containing the dataset pointed to by + the OpenDAP URL. + + :raises: ServerError + ''' + #Downloading the dataset using podaac toolkit + podaac = Podaac() + path = os.path.dirname(os.path.abspath(__file__)) + granuleName = podaac.extract_l4_granule(datasetId=datasetId, shortName=datasetShortName, path=path) + path = path+'/'+granuleName + d = netcdf_dataset(path, mode='r') + dataset = d.variables[variable] + + # By convention, but not by standard, if the dimensions exist, they will be in the order: + # time (t), altitude (z), latitude (y), longitude (x) + # but conventions aren't always followed and all dimensions aren't always present so + # see if we can make some educated deductions before defaulting to just pulling the first three + # columns. + temp_dimensions = map(lambda x:x.lower(),dataset.dimensions) + dataset_dimensions = dataset.dimensions + time = dataset_dimensions[temp_dimensions.index('time') if 'time' in temp_dimensions else 0] + lat = dataset_dimensions[temp_dimensions.index('lat') if 'lat' in temp_dimensions else 1] + lon = dataset_dimensions[temp_dimensions.index('lon') if 'lon' in temp_dimensions else 2] + + # Time is given to us in some units since an epoch. We need to convert + # these values to datetime objects. Note that we use the main object's + # time object and not the dataset specific reference to it. We need to + # grab the 'units' from it and it fails on the dataset specific object. + times = np.array(_convert_times_to_datetime(d[time])) + lats = np.array(d.variables[lat][:]) + lons = np.array(d.variables[lon][:]) + values = np.array(dataset[:]) + origin = { + 'source' : 'PO.DAAC', + 'url' : 'podaac.jpl.nasa.gov/ws' + } + + # Removing the downloaded temporary granule before creating the OCW dataset. + d.close() + path = os.path.join(os.path.dirname(__file__), granuleName) + os.remove(path) + + return Dataset(lats, lons, times, values, variable, name=name, origin=origin) http://git-wip-us.apache.org/repos/asf/climate/blob/116d16f0/ocw/tests/test_podaac.py ---------------------------------------------------------------------- diff --git a/ocw/tests/test_podaac.py b/ocw/tests/test_podaac.py new file mode 100644 index 0000000..d0c2808 --- /dev/null +++ b/ocw/tests/test_podaac.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import ocw.data_source.podaac as podaac +import unittest, os +from ocw.dataset import Dataset + +class TestPodaacDataSource(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.datasetId = 'PODAAC-GHCMC-4FM02' + cls.datasetShortName = 'CMC0.2deg-CMC-L4-GLOB-v2.0' + cls.variable = 'sea_ice_fraction' + cls.name = 'PO.DAAC_test_dataset' + cls.file_path = os.path.dirname(os.path.abspath(__file__)) + cls.format = '.nc' + cls.dataset = podaac.load_dataset(cls.variable, cls.datasetId, cls.datasetShortName, cls.name) + + def test_is_dataset(self): + self.assertTrue(isinstance(self.dataset, Dataset)) + + def test_dataset_lats(self): + self.assertEquals(len(self.dataset.lats), 901) + + def test_dataset_lons(self): + self.assertEquals(len(self.dataset.lons), 1800) + + def test_dataset_times(self): + self.assertEquals(len(self.dataset.times), 1) + + def test_dataset_values(self): + self.assertEquals(len(self.dataset.values),1) + + def test_valid_date_conversion(self): + start = dt.datetime(2006, 6, 7, 12) + self.assertTrue(start == self.dataset.times[0]) + + def test_dataset_origin(self): + self.assertEquals(self.dataset.origin['source'], 'PO.DAAC') + self.assertEquals(self.dataset.origin['url'], 'podaac.jpl.nasa.gov/ws') + + def test_custom_name(self): + self.assertEquals(self.dataset.name, self.name) + + +if __name__ == '__main__': + unittest.main()
