woot thanks Denis! ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Chris Mattmann, Ph.D. Chief Architect Instrument Software and Science Data Systems Section (398) NASA Jet Propulsion Laboratory Pasadena, CA 91109 USA Office: 168-519, Mailstop: 168-527 Email: [email protected] WWW: http://sunset.usc.edu/~mattmann/ ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Adjunct Associate Professor, Computer Science Department University of Southern California, Los Angeles, CA 90089 USA ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-----Original Message----- From: "[email protected]" <[email protected]> Reply-To: "[email protected]" <[email protected]> Date: Tuesday, September 30, 2014 at 12:51 PM To: "[email protected]" <[email protected]> Subject: git commit: Add Resources sheet into excel spreadsheet. It is no longer necessary to have ".rc" file Add -x option in obs4Mips.py in the "file_template" it is now possible to specify the variable to pass a a parameter. "trmm_%s.lst, years" where years c >Repository: climate >Updated Branches: > refs/heads/master 8469bcacb -> 7a0a7e2d4 > > >Add Resources sheet into excel spreadsheet. It is no longer necessary to >have ".rc" file >Add -x option in obs4Mips.py >in the "file_template" it is now possible to specify the variable to pass >a a parameter. "trmm_%s.lst,years" where years contain the values to >pass into the format "%s". > > >Project: http://git-wip-us.apache.org/repos/asf/climate/repo >Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/7a0a7e2d >Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/7a0a7e2d >Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/7a0a7e2d > >Branch: refs/heads/master >Commit: 7a0a7e2d4b52ab7573787425351999f7ab6d90ce >Parents: 8469bca >Author: Nadeau <[email protected]> >Authored: Tue Sep 30 15:43:30 2014 -0400 >Committer: Nadeau <[email protected]> >Committed: Tue Sep 30 15:43:30 2014 -0400 > >---------------------------------------------------------------------- > obs4MIPs/INSTALLATION | 3 + > obs4MIPs/README | 2 + > obs4MIPs/Toolbox/ESGFexcel.py | 125 +++++++++++++++++++ > .../examples/ECMWF/ecmwf_table_obs4MIPs.xls | Bin 10752 -> 55296 >bytes > obs4MIPs/obs4MIPs_process.py | 29 +++-- > 5 files changed, 152 insertions(+), 7 deletions(-) >---------------------------------------------------------------------- > > >http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/INST >ALLATION >---------------------------------------------------------------------- >diff --git a/obs4MIPs/INSTALLATION b/obs4MIPs/INSTALLATION >index 082b787..e647d55 100644 >--- a/obs4MIPs/INSTALLATION >+++ b/obs4MIPs/INSTALLATION >@@ -54,6 +54,9 @@ http://code.google.com/p/netcdf4-python/ > > RESSOURCE file documentation > ============================= >+********************** >+*** NOTE: The Resource file can now be including in the Excel >spreadsheet as a new sheet called: Resources. See >"ecmwf_table_obs4MIPs.xls" for details. >+********************** > > In order to run osb4MIPs, the first thing to do is to create a resource >file giving information about your original data. The resource files >contain all Global Attributes and information needed to convert a >variable to a CMIP5 variable. > > >http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/READ >ME >---------------------------------------------------------------------- >diff --git a/obs4MIPs/README b/obs4MIPs/README >index f9983b8..c0cc05f 100644 >--- a/obs4MIPs/README >+++ b/obs4MIPs/README >@@ -12,6 +12,8 @@ obs4MIPs.py -r trmm.rc > > Convert trmm data to CMIP5 format using excel spreadsheet > obs4MIPs.py -r ecmwf.rc >+or >+obs4MIPs.py -x ecmwf_table_obs4MIPs.xls > > -------------- > Installation > >http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/Tool >box/ESGFexcel.py >---------------------------------------------------------------------- >diff --git a/obs4MIPs/Toolbox/ESGFexcel.py b/obs4MIPs/Toolbox/ESGFexcel.py >new file mode 100644 >index 0000000..92fd4df >--- /dev/null >+++ b/obs4MIPs/Toolbox/ESGFexcel.py >@@ -0,0 +1,125 @@ >+# Licensed to the Apache Software Foundation (ASF) under one >+# or more contributor license agreements. See the NOTICE file >+# distributed with this work for additional information >+# regarding copyright ownership. The ASF licenses this file >+# to you under the Apache License, Version 2.0 (the >+# "License"); you may not use this file except in compliance >+# with the License. You may obtain a copy of the License at >+# >+# http://www.apache.org/licenses/LICENSE-2.0 >+# >+# Unless required by applicable law or agreed to in writing, >+# software distributed under the License is distributed on an >+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY >+# KIND, either express or implied. See the License for the >+# specific language governing permissions and limitations >+# under the License. >+ >+ >+import pdb >+import xlrd >+import shutil >+import os,sys >+from numpy import arange >+from Toolbox.CMORresources import CMORAttributes >+ >+# ******************************************************************** >+# ESGFexcel() >+# >+# ******************************************************************** >+class ESGFexcel: >+ ''' >+ Create dictionary based on a file using key=value standard from Excel >+ Sheet resource Key is in Column A, Value is in Column B. >+ ''' >+ def __init__(self,excelFile): >+ ''' >+ ''' >+ self.xcl = excelFile >+ if( os.path.isfile(self.xcl) ): >+ wb=xlrd.open_workbook(self.xcl) >+ else: >+ print "****** Could not find "+self.xcl+" file ****" >+ print "****** Please check excel file name ****" >+ raise NameError(self.xcl) >+ >+ sheet=wb.sheet_by_name('Resources') >+ >+ for i in arange(sheet.nrows-1) + 1: >+ key = sheet.row(i)[0] >+ value = sheet.row(i)[1] >+ >+ self.resources = dict( [( key, value, ) >+ for( key, value ) in >+ [ (sheet.row(i)[0].value, >str(sheet.row(i)[1].value)) >+ for i in arange(sheet.nrows-1) + 1] ] ) >+ pdb.set_trace() >+ self.ReadXCL() >+ print self.resources.keys() >+ >+ def ReadXCL(self): >+ ''' >+ Read Excel Table and fill rc variable related field. >+ ''' >+ try: >+ import xlrd >+ except: >+ print "****** Could not find xlrd Python Package ****" >+ print "****** Please install xlrd package to read excel files >****" >+ >+ if( os.path.isfile(self.xcl) ): >+ wb=xlrd.open_workbook(self.xcl) >+ else: >+ print "****** Could not find "+self.xcl+" file ****" >+ print "****** Please check excel file name ****" >+ raise NameError(self.xcl) >+ >+ sheet=wb.sheet_by_name('Variables') >+ >+ self.resources['cmor_var'] = [ sheet.row( i )[ 0 ].value.\ >+ encode('ascii','ignore') \ >+ for i in arange(sheet.nrows-2) >+ 2 ] >+ >+ self.resources['original_var'] = [ sheet.row( i )[ 1 ].value.\ >+ encode('ascii','ignore') \ >+ for i in >arange(sheet.nrows-2) + 2 ] >+ >+ self.resources['original_units']=[ sheet.row( i )[ 2 ].value.\ >+ encode('ascii','ignore') \ >+ for i in >arange(sheet.nrows-2) + 2 ] >+ >+ self.resources['level'] =[ sheet.row( i )[ 3 ].value.\ >+ encode('ascii','ignore') \ >+ for i in >arange(sheet.nrows-2) + 2] >+ >+ self.resources['equation'] =[ sheet.row( i )[ 6 ].value.\ >+ encode('ascii','ignore') \ >+ for i in >arange(sheet.nrows-2) + 2 ] >+ >+ # >----------------------------------------------------------------- >+ # Make sure it is a string. The main program will call eval on >it. >+ # >----------------------------------------------------------------- >+ self.resources['cmor_var'] = >str(self.resources['cmor_var'] ) >+ self.resources['original_var'] = >str(self.resources['original_var']) >+ self.resources['original_units'] >=str(self.resources['original_units']) >+ self.resources['equation'] =str(self.resources['equation']) >+ self.resources['level'] =str(self.resources['level']) >+ return 1 >+ >+ def __getitem__(self,key): >+ ''' >+ Retreive item from resource dictionary >+ ''' >+ return self.resources[key] >+ >+ def __setitem__(self,key,value): >+ ''' >+ ''' >+ self.resources[key]=value >+ >+ def __delete__(self,key): >+ ''' >+ ''' >+ del self.resources[key] >+ >+ > >http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/exam >ples/ECMWF/ecmwf_table_obs4MIPs.xls >---------------------------------------------------------------------- >diff --git a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls >b/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls >index 271c2a7..d2d12d7 100644 >Binary files a/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls and >b/obs4MIPs/examples/ECMWF/ecmwf_table_obs4MIPs.xls differ > >http://git-wip-us.apache.org/repos/asf/climate/blob/7a0a7e2d/obs4MIPs/obs4 >MIPs_process.py >---------------------------------------------------------------------- >diff --git a/obs4MIPs/obs4MIPs_process.py b/obs4MIPs/obs4MIPs_process.py >index bb6d238..3370100 100755 >--- a/obs4MIPs/obs4MIPs_process.py >+++ b/obs4MIPs/obs4MIPs_process.py >@@ -26,6 +26,7 @@ import factory > import numpy > from factory.formats import import_equation > from Toolbox.ESGFresources import * >+from Toolbox.ESGFexcel import * > from Toolbox.CMORresources import CMORTable > > >@@ -38,11 +39,18 @@ def process( rc ): > ''' > Convert netcdf/matlab/grads files into CMIP5 format. > ''' >- #pdb.set_trace() >+ pdb.set_trace() > # ---------------------------- > # Loop yearly on file list. > # ---------------------------- >- for year in rc['years'].split(","): >+ file_template = rc['file_template'].split(","); >+ if( len(file_template) == 2 ): >+ template_parameter = file_template[1] >+ rc['file_template'] = file_template[0] >+ else: >+ template_parameter = 'years' >+ >+ for year in rc[template_parameter].split(","): > if(year == ''): > files= os.popen( "ls " + rc['file_template'] ).readlines() > else: >@@ -419,9 +427,10 @@ def usage(message): > def main(): > ''' > ''' >+ pdb.set_trace() > try: >- opts, args = getopt.getopt(sys.argv[1:], "hy:r:", >- ["help" ,"year=","resource="]) >+ opts, args = getopt.getopt(sys.argv[1:], "hy:r:x:", >+ ["help" >,"year=","resource=","excel="]) > except getopt.GetoptError, err: > usage(str(err))# will print something like "option -a not >recognized" > return(2) >@@ -431,9 +440,12 @@ def main(): > # -------------------------- > year = -1 > resource = None >+ excel = None > for o, a in opts: > if o in ("-r", "--resource"): > resource = a >+ elif o in ("-x", "--excel"): >+ excel = a > elif o in ("-h", "--help"): > usage() > return(0) >@@ -445,14 +457,17 @@ def main(): > # ------------------------------ > # Does the resource file exist? > # ------------------------------ >- if( resource == None ) or ( not os.path.isfile( resource ) ) : >- usage("bad Input Resource File") >+ if( ((resource == None ) or ( not os.path.isfile( resource ) )) and >(( excel == None ) or ( not os.path.isfile( excel ) )) ): >+ usage("bad Input Resource/Excel File") > return 1 > > # ----------------------- > # Read in "rc" file > # ----------------------- >- rc=ESGFresources(resource) >+ if( resource ): >+ rc = ESGFresources( resource ) >+ if( excel ): >+ rc = ESGFexcel( excel ) > > # -------------------------------- > # Extract CMIP5 Table information >
