Repository: climate Updated Branches: refs/heads/master ddae7589c -> 625eec2d2
Fixed issues with ESGF loader Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/625eec2d Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/625eec2d Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/625eec2d Branch: refs/heads/master Commit: 625eec2d2454d699e9a0dfe4e9d9a73f6c55ed57 Parents: ddae758 Author: Alex Goodman <ago...@users.noreply.github.com> Authored: Tue Aug 9 15:56:28 2016 -0700 Committer: Alex Goodman <ago...@users.noreply.github.com> Committed: Tue Aug 9 15:56:28 2016 -0700 ---------------------------------------------------------------------- .../ESGF_example/ESGF-Example.yaml | 4 +- RCMES/run_RCMES.py | 46 +++++++++++--------- ocw/data_source/esgf.py | 10 ++--- ocw/esgf/logon.py | 6 +-- 4 files changed, 33 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/625eec2d/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml ---------------------------------------------------------------------- diff --git a/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml b/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml index eb65a3d..29d06c8 100644 --- a/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml +++ b/RCMES/configuration_files/ESGF_example/ESGF-Example.yaml @@ -39,13 +39,13 @@ regrid: datasets: reference: - - loader_name: ESGF + - loader_name: esgf name: AVISO dataset_id: obs4MIPs.CNES.AVISO.zos.mon.v20110829|esgf-data.jpl.nasa.gov variable_name: zos targets: - - loader_name: ESGF + - loader_name: esgf name: MPI-ESM dataset_id: cmip5.output1.MPI-M.MPI-ESM-LR.decadal1994.mon.ocean.Omon.r1i1p1.v20120529|esgf1.dkrz.de variable_name: zos http://git-wip-us.apache.org/repos/asf/climate/blob/625eec2d/RCMES/run_RCMES.py ---------------------------------------------------------------------- diff --git a/RCMES/run_RCMES.py b/RCMES/run_RCMES.py index f07f2ef..65d5e3c 100644 --- a/RCMES/run_RCMES.py +++ b/RCMES/run_RCMES.py @@ -30,28 +30,29 @@ from ocw.dataset import Bounds from ocw.dataset_loader import DatasetLoader from metrics_and_plots import * -def load_datasets_from_config(*loader_options, **kwargs): +def load_datasets_from_config(extra_opts, *loader_opts): ''' Generic dataset loading function. ''' - for opt in loader_options: + for opt in loader_opts: loader_name = opt['loader_name'] - if loader_name == 'ESGF': - if password is None: - username=raw_input('Enter your ESGF OpenID:\n') - password=getpass(prompt='Enter your ESGF password:\n') - - opt['username'] = username - opt['password'] = password + if loader_name == 'esgf': + if extra_opts['password'] is None: + extra_opts['username'] = raw_input('Enter your ESGF OpenID:\n') + extra_opts['password'] = getpass( + prompt='Enter your ESGF password:\n') + + opt['esgf_username'] = extra_opts['username'] + opt['esgf_password'] = extra_opts['password'] elif loader_name == 'rcmed': - opt['min_lat'] = kwargs['min_lat'] - opt['max_lat'] = kwargs['max_lat'] - opt['min_lon'] = kwargs['min_lon'] - opt['max_lon'] = kwargs['max_lon'] - opt['start_time'] = kwargs['start_time'] - opt['end_time'] = kwargs['end_time'] - - loader = DatasetLoader(*loader_options) + opt['min_lat'] = extra_opts['min_lat'] + opt['max_lat'] = extra_opts['max_lat'] + opt['min_lon'] = extra_opts['min_lon'] + opt['max_lon'] = extra_opts['max_lon'] + opt['start_time'] = extra_opts['start_time'] + opt['end_time'] = extra_opts['end_time'] + + loader = DatasetLoader(*loader_opts) loader.load_datasets() return loader.datasets @@ -75,8 +76,11 @@ min_lat = space_info['min_lat'] max_lat = space_info['max_lat'] min_lon = space_info['min_lon'] max_lon = space_info['max_lon'] -kwargs = {'min_lat': min_lat, 'max_lat': max_lat, 'min_lon': min_lon, - 'max_lon': max_lon, 'start_time': start_time, 'end_time': end_time} + +# Additional arguments for the DatasetLoader +extra_opts = {'min_lat': min_lat, 'max_lat': max_lat, 'min_lon': min_lon, + 'max_lon': max_lon, 'start_time': start_time, + 'end_time': end_time, 'username': None, 'password': None} # Get the dataset loader options obs_data_info = config['datasets']['reference'] @@ -98,7 +102,7 @@ for i, info in enumerate(model_data_info): """ Step 1: Load the observation data """ print 'Loading observation datasets:\n',obs_data_info -obs_datasets = load_datasets_from_config(*obs_data_info, **kwargs) +obs_datasets = load_datasets_from_config(extra_opts, *obs_data_info) obs_names = [dataset.name for dataset in obs_datasets] for i, dataset in enumerate(obs_datasets): if temporal_resolution == 'daily' or temporal_resolution == 'monthly': @@ -109,7 +113,7 @@ for i, dataset in enumerate(obs_datasets): obs_dataset.values *= multiplying_factor[i] """ Step 2: Load model NetCDF Files into OCW Dataset Objects """ -model_datasets = load_datasets_from_config(*model_data_info, **kwargs) +model_datasets = load_datasets_from_config(extra_opts, *model_data_info) model_names = [dataset.name for dataset in model_datasets] if temporal_resolution == 'daily' or temporal_resolution == 'monthly': for i, dataset in enumerate(model_datasets): http://git-wip-us.apache.org/repos/asf/climate/blob/625eec2d/ocw/data_source/esgf.py ---------------------------------------------------------------------- diff --git a/ocw/data_source/esgf.py b/ocw/data_source/esgf.py index 3ea559f..ac802c0 100644 --- a/ocw/data_source/esgf.py +++ b/ocw/data_source/esgf.py @@ -30,7 +30,7 @@ from bs4 import BeautifulSoup import requests def load_dataset(dataset_id, - variable, + variable_name, esgf_username, esgf_password, search_url=DEFAULT_ESGF_SEARCH, @@ -43,8 +43,8 @@ def load_dataset(dataset_id, :param dataset_id: The ESGF ID of the dataset to load. :type dataset_id: :mod:`string` - :param variable: The variable to load. - :type variable: :mod:`string` + :param variable_name: The variable to load. + :type variable_name: :mod:`string` :param esgf_username: ESGF OpenID value to use for authentication. :type esgf_username: :mod:`string` @@ -79,7 +79,7 @@ def load_dataset(dataset_id, ''' download_data = _get_file_download_data(url=search_url, dataset_id=dataset_id, - variable=variable) + variable=variable_name) datasets = [] for url, var in download_data: @@ -97,7 +97,7 @@ def load_dataset(dataset_id, origin = { 'source': 'esgf', 'dataset_id': dataset_id, - 'variable': variable + 'variable': variable_name } for ds in datasets: ds.origin = origin http://git-wip-us.apache.org/repos/asf/climate/blob/625eec2d/ocw/esgf/logon.py ---------------------------------------------------------------------- diff --git a/ocw/esgf/logon.py b/ocw/esgf/logon.py index 031e968..8b4c034 100644 --- a/ocw/esgf/logon.py +++ b/ocw/esgf/logon.py @@ -32,16 +32,12 @@ def logon(openid, password): The certificate is written in the location ~/.esg/credentials.pem. The trusted CA certificates are written in the directory ~/.esg/certificates. ''' - # Must configure the DN of the JPL MyProxy server if using a JPL openid if JPL_HOSTNAME in openid: os.environ['MYPROXY_SERVER_DN'] = JPL_MYPROXY_SERVER_DN lm = LogonManager() - lm.logon_with_openid(openid, password) + lm.logon_with_openid(openid, password, bootstrap=True) return lm.is_logged_on() - - -
