Repository: climate Updated Branches: refs/heads/master 8469bcacb -> 7a0a7e2d4
Add Resources sheet into excel spreadsheet. It is no longer necessary to have ".rc" file Add -x option in obs4Mips.py in the "file_template" it is now possible to specify the variable to pass a a parameter. "trmm_%s.lst,years" where years contain the values to pass into the format "%s". Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/7a0a7e2d Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/7a0a7e2d Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/7a0a7e2d Branch: refs/heads/master Commit: 7a0a7e2d4b52ab7573787425351999f7ab6d90ce Parents: 8469bca Author: Nadeau <[email protected]> Authored: Tue Sep 30 15:43:30 2014 -0400 Committer: Nadeau <[email protected]> Committed: Tue Sep 30 15:43:30 2014 -0400 ---------------------------------------------------------------------- obs4MIPs/INSTALLATION | 3 + obs4MIPs/README | 2 + obs4MIPs/Toolbox/ESGFexcel.py | 125 +++++++++++++++++++ .../examples/ECMWF/ecmwf_table_obs4MIPs.xls | Bin 10752 -> 55296 bytes obs4MIPs/obs4MIPs_process.py | 29 +++-- 5 files changed, 152 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/INSTALLATION ---------------------------------------------------------------------- diff --git a/obs4MIPs/INSTALLATION b/obs4MIPs/INSTALLATION index 082b787..e647d55 100644 --- a/obs4MIPs/INSTALLATION +++ b/obs4MIPs/INSTALLATION @@ -54,6 +54,9 @@ http://code.google.com/p/netcdf4-python/ RESSOURCE file documentation ============================= +********************** +*** NOTE: The Resource file can now be including in the Excel spreadsheet as a new sheet called: Resources. See "ecmwf_table_obs4MIPs.xls" for details. +********************** In order to run osb4MIPs, the first thing to do is to create a resource file giving information about your original data. The resource files contain all Global Attributes and information needed to convert a variable to a CMIP5 variable. http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/README ---------------------------------------------------------------------- diff --git a/obs4MIPs/README b/obs4MIPs/README index f9983b8..c0cc05f 100644 --- a/obs4MIPs/README +++ b/obs4MIPs/README @@ -12,6 +12,8 @@ obs4MIPs.py -r trmm.rc Convert trmm data to CMIP5 format using excel spreadsheet obs4MIPs.py -r ecmwf.rc +or +obs4MIPs.py -x ecmwf_table_obs4MIPs.xls -------------- Installation http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/Toolbox/ESGFexcel.py ---------------------------------------------------------------------- diff --git a/obs4MIPs/Toolbox/ESGFexcel.py b/obs4MIPs/Toolbox/ESGFexcel.py new file mode 100644 index 0000000..92fd4df --- /dev/null +++ b/obs4MIPs/Toolbox/ESGFexcel.py @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import pdb +import xlrd +import shutil +import os,sys +from numpy import arange +from Toolbox.CMORresources import CMORAttributes + +# ******************************************************************** +# ESGFexcel() +# +# ******************************************************************** +class ESGFexcel: + ''' + Create dictionary based on a file using key=value standard from Excel + Sheet resource Key is in Column A, Value is in Column B. + ''' + def __init__(self,excelFile): + ''' + ''' + self.xcl = excelFile + if( os.path.isfile(self.xcl) ): + wb=xlrd.open_workbook(self.xcl) + else: + print "****** Could not find "+self.xcl+" file ****" + print "****** Please check excel file name ****" + raise NameError(self.xcl) + + sheet=wb.sheet_by_name('Resources') + + for i in arange(sheet.nrows-1) + 1: + key = sheet.row(i)[0] + value = sheet.row(i)[1] + + self.resources = dict( [( key, value, ) + for( key, value ) in + [ (sheet.row(i)[0].value, str(sheet.row(i)[1].value)) + for i in arange(sheet.nrows-1) + 1] ] ) + pdb.set_trace() + self.ReadXCL() + print self.resources.keys() + + def ReadXCL(self): + ''' + Read Excel Table and fill rc variable related field. + ''' + try: + import xlrd + except: + print "****** Could not find xlrd Python Package ****" + print "****** Please install xlrd package to read excel files ****" + + if( os.path.isfile(self.xcl) ): + wb=xlrd.open_workbook(self.xcl) + else: + print "****** Could not find "+self.xcl+" file ****" + print "****** Please check excel file name ****" + raise NameError(self.xcl) + + sheet=wb.sheet_by_name('Variables') + + self.resources['cmor_var'] = [ sheet.row( i )[ 0 ].value.\ + encode('ascii','ignore') \ + for i in arange(sheet.nrows-2) + 2 ] + + self.resources['original_var'] = [ sheet.row( i )[ 1 ].value.\ + encode('ascii','ignore') \ + for i in arange(sheet.nrows-2) + 2 ] + + self.resources['original_units']=[ sheet.row( i )[ 2 ].value.\ + encode('ascii','ignore') \ + for i in arange(sheet.nrows-2) + 2 ] + + self.resources['level'] =[ sheet.row( i )[ 3 ].value.\ + encode('ascii','ignore') \ + for i in arange(sheet.nrows-2) + 2] + + self.resources['equation'] =[ sheet.row( i )[ 6 ].value.\ + encode('ascii','ignore') \ + for i in arange(sheet.nrows-2) + 2 ] + + # ----------------------------------------------------------------- + # Make sure it is a string. The main program will call eval on it. + # ----------------------------------------------------------------- + self.resources['cmor_var'] = str(self.resources['cmor_var'] ) + self.resources['original_var'] = str(self.resources['original_var']) + self.resources['original_units'] =str(self.resources['original_units']) + self.resources['equation'] =str(self.resources['equation']) + self.resources['level'] =str(self.resources['level']) + return 1 + + def __getitem__(self,key): + ''' + Retreive item from resource dictionary + ''' + return self.resources[key] + + def __setitem__(self,key,value): + ''' + ''' + self.resources[key]=value + + def __delete__(self,key): + ''' + ''' + del self.resources[key] + + http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls ---------------------------------------------------------------------- diff --git a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls b/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls index 271c2a7..d2d12d7 100644 Binary files a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls and b/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls differ http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/obs4MIPs_process.py ---------------------------------------------------------------------- diff --git a/obs4MIPs/obs4MIPs_process.py b/obs4MIPs/obs4MIPs_process.py index bb6d238..3370100 100755 --- a/obs4MIPs/obs4MIPs_process.py +++ b/obs4MIPs/obs4MIPs_process.py @@ -26,6 +26,7 @@ import factory import numpy from factory.formats import import_equation from Toolbox.ESGFresources import * +from Toolbox.ESGFexcel import * from Toolbox.CMORresources import CMORTable @@ -38,11 +39,18 @@ def process( rc ): ''' Convert netcdf/matlab/grads files into CMIP5 format. ''' - #pdb.set_trace() + pdb.set_trace() # ---------------------------- # Loop yearly on file list. # ---------------------------- - for year in rc['years'].split(","): + file_template = rc['file_template'].split(","); + if( len(file_template) == 2 ): + template_parameter = file_template[1] + rc['file_template'] = file_template[0] + else: + template_parameter = 'years' + + for year in rc[template_parameter].split(","): if(year == ''): files= os.popen( "ls " + rc['file_template'] ).readlines() else: @@ -419,9 +427,10 @@ def usage(message): def main(): ''' ''' + pdb.set_trace() try: - opts, args = getopt.getopt(sys.argv[1:], "hy:r:", - ["help" ,"year=","resource="]) + opts, args = getopt.getopt(sys.argv[1:], "hy:r:x:", + ["help" ,"year=","resource=","excel="]) except getopt.GetoptError, err: usage(str(err))# will print something like "option -a not recognized" return(2) @@ -431,9 +440,12 @@ def main(): # -------------------------- year = -1 resource = None + excel = None for o, a in opts: if o in ("-r", "--resource"): resource = a + elif o in ("-x", "--excel"): + excel = a elif o in ("-h", "--help"): usage() return(0) @@ -445,14 +457,17 @@ def main(): # ------------------------------ # Does the resource file exist? # ------------------------------ - if( resource == None ) or ( not os.path.isfile( resource ) ) : - usage("bad Input Resource File") + if( ((resource == None ) or ( not os.path.isfile( resource ) )) and (( excel == None ) or ( not os.path.isfile( excel ) )) ): + usage("bad Input Resource/Excel File") return 1 # ----------------------- # Read in "rc" file # ----------------------- - rc=ESGFresources(resource) + if( resource ): + rc = ESGFresources( resource ) + if( excel ): + rc = ESGFexcel( excel ) # -------------------------------- # Extract CMIP5 Table information
