Merge branch 'master' into CLIMATE-912 Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/51d9dce1 Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/51d9dce1 Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/51d9dce1
Branch: refs/heads/master Commit: 51d9dce12c9c9b77e7e48f3cca054bcf5647e458 Parents: 5c86f3a 56989f5 Author: Lewis John McGibbney <lewis.mcgibb...@gmail.com> Authored: Wed Aug 9 11:42:08 2017 -0700 Committer: GitHub <nore...@github.com> Committed: Wed Aug 9 11:42:08 2017 -0700 ---------------------------------------------------------------------- RCMES/cli_app.py | 16 +- .../NARCCAP_examples/Fig14_and_Fig15.yaml | 4 +- .../NARCCAP_examples/Fig16_summer.yaml | 5 +- .../NARCCAP_examples/Fig16_winter.yaml | 4 +- .../run_statistical_downscaling.py | 17 +- examples/esgf_integration_example.py | 17 +- examples/podaac_integration_example.py | 7 +- examples/simple_model_to_model_bias.py | 14 +- examples/taylor_diagram_example.py | 13 +- examples/time_series_with_regions.py | 21 +- mccsearch/code/mccSearch.py | 16 +- mccsearch/code/mccSearchUI.py | 1 + ocw/data_source/esgf.py | 11 +- ocw/data_source/local.py | 4 +- ocw/dataset_processor.py | 90 ++---- ocw/esgf/download.py | 28 +- ocw/tests/TestGetNetcdfVariableNames.nc | Bin 0 -> 471996 bytes ocw/tests/test_dataset_processor.py | 6 +- ocw/tests/test_local.py | 280 +++++++++++-------- ocw/utils.py | 2 +- test_smoke.py | 4 +- 21 files changed, 306 insertions(+), 254 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/51d9dce1/mccsearch/code/mccSearch.py ---------------------------------------------------------------------- diff --cc mccsearch/code/mccSearch.py index 9f396f8,6f1fa26..c63b243 --- a/mccsearch/code/mccSearch.py +++ b/mccsearch/code/mccSearch.py @@@ -44,72 -44,61 +44,73 @@@ import ocw.plotter as plotte #----------------------- GLOBAL VARIABLES -------------------------- # --------------------- User defined variables --------------------- -#FYI the lat lon values are not necessarily inclusive of the points given. These are the limits -#the first point closest the the value (for the min) from the MERG data is used, etc. -LATMIN = '5.0' #min latitude; -ve values in the SH e.g. 5S = -5 -LATMAX = '19.0' #max latitude; -ve values in the SH e.g. 5S = -5 20.0 -LONMIN = '-5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30 -LONMAX = '5.0' #min longitude; -ve values in the WH e.g. 59.8W = -59.8 30 -XRES = 4.0 #x direction spatial resolution in km -YRES = 4.0 #y direction spatial resolution in km -TRES = 1 #temporal resolution in hrs -LAT_DISTANCE = 111.0 #the avg distance in km for 1deg lat for the region being considered -LON_DISTANCE = 111.0 #the avg distance in km for 1deg lon for the region being considered -STRUCTURING_ELEMENT = [[0,1,0],[1,1,1],[0,1,0]] #the matrix for determining the pattern for the contiguous boxes and must - #have same rank of the matrix it is being compared against -#criteria for determining cloud elements and edges -T_BB_MAX = 243 #warmest temp to allow (-30C to -55C according to Morel and Sensi 2002) -T_BB_MIN = 218 #cooler temp for the center of the system -CONVECTIVE_FRACTION = 0.90 #the min temp/max temp that would be expected in a CE.. this is highly conservative (only a 10K difference) -MIN_MCS_DURATION = 3 #minimum time for a MCS to exist -AREA_MIN = 2400.0 #minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400 -MIN_OVERLAP= 10000.00 #km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992 +# FYI the lat lon values are not necessarily inclusive of the points given. These are the limits +# the first point closest the the value (for the min) from the MERG data +# is used, etc. +LATMIN = '5.0' # min latitude; -ve values in the SH e.g. 5S = -5 +LATMAX = '19.0' # max latitude; -ve values in the SH e.g. 5S = -5 20.0 +LONMIN = '-5.0' # min longitude; -ve values in the WH e.g. 59.8W = -59.8 -30 +LONMAX = '5.0' # min longitude; -ve values in the WH e.g. 59.8W = -59.8 30 +XRES = 4.0 # x direction spatial resolution in km +YRES = 4.0 # y direction spatial resolution in km +TRES = 1 # temporal resolution in hrs +LAT_DISTANCE = 111.0 # the avg distance in km for 1deg lat for the region being considered +LON_DISTANCE = 111.0 # the avg distance in km for 1deg lon for the region being considered +# the matrix for determining the pattern for the contiguous boxes and must +STRUCTURING_ELEMENT = [[0, 1, 0], [1, 1, 1], [0, 1, 0]] +# have same rank of the matrix it is being compared against +# criteria for determining cloud elements and edges +# warmest temp to allow (-30C to -55C according to Morel and Sensi 2002) +T_BB_MAX = 243 +T_BB_MIN = 218 # cooler temp for the center of the system +# the min temp/max temp that would be expected in a CE.. this is highly +# conservative (only a 10K difference) +CONVECTIVE_FRACTION = 0.90 +MIN_MCS_DURATION = 3 # minimum time for a MCS to exist +# minimum area for CE criteria in km^2 according to Vila et al. (2008) is 2400 +AREA_MIN = 2400.0 +# km^2 from Williams and Houze 1987, indir ref in Arnaud et al 1992 +MIN_OVERLAP = 10000.00 #---the MCC criteria -ECCENTRICITY_THRESHOLD_MAX = 1.0 #tending to 1 is a circle e.g. hurricane, -ECCENTRICITY_THRESHOLD_MIN = 0.50 #tending to 0 is a linear e.g. squall line -OUTER_CLOUD_SHIELD_AREA = 80000.0 #km^2 -INNER_CLOUD_SHIELD_AREA = 30000.0 #km^2 -OUTER_CLOUD_SHIELD_TEMPERATURE = 233 #in K -INNER_CLOUD_SHIELD_TEMPERATURE = 213 #in K -MINIMUM_DURATION = 6 #min number of frames the MCC must exist for (assuming hrly frames, MCCs is 6hrs) -MAXIMUM_DURATION = 24#max number of framce the MCC can last for +ECCENTRICITY_THRESHOLD_MAX = 1.0 # tending to 1 is a circle e.g. hurricane, +ECCENTRICITY_THRESHOLD_MIN = 0.50 # tending to 0 is a linear e.g. squall line +OUTER_CLOUD_SHIELD_AREA = 80000.0 # km^2 +INNER_CLOUD_SHIELD_AREA = 30000.0 # km^2 +OUTER_CLOUD_SHIELD_TEMPERATURE = 233 # in K +INNER_CLOUD_SHIELD_TEMPERATURE = 213 # in K +# min number of frames the MCC must exist for (assuming hrly frames, MCCs +# is 6hrs) +MINIMUM_DURATION = 6 +MAXIMUM_DURATION = 24 # max number of framce the MCC can last for #------------------- End user defined Variables ------------------- -edgeWeight = [1,2,3] #weights for the graph edges -#graph object fo the CEs meeting the criteria +edgeWeight = [1, 2, 3] # weights for the graph edges +# graph object fo the CEs meeting the criteria CLOUD_ELEMENT_GRAPH = nx.DiGraph() -#graph meeting the CC criteria +# graph meeting the CC criteria PRUNED_GRAPH = nx.DiGraph() #------------------------ End GLOBAL VARS ------------------------- ++ #************************ Begin Functions ************************* #****************************************************************** -def readMergData(dirname, filelist = None): + + +def readMergData(dirname, filelist=None): ''' Purpose:: - Read MERG data into RCMES format - + Read MERG data into RCMES format + Input:: - dirname: a string representing the directory to the MERG files in NETCDF format - filelist (optional): a list of strings representing the filenames betweent the start and end dates provided - + dirname: a string representing the directory to the MERG files in NETCDF format + filelist (optional): a list of strings representing the filenames betweent the start and end dates provided + Output:: - A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature - criteria for each frame + A 3D masked array (t,lat,lon) with only the variables which meet the minimum temperature + criteria for each frame Assumptions:: - The MERG data has been converted to NETCDF using LATS4D - The data has the same lat/lon format + The MERG data has been converted to NETCDF using LATS4D + The data has the same lat/lon format TODO:: figure out how to use netCDF4 to do the clipping tmp = netCDF4.Dataset(filelist[0]) @@@ -123,17 -112,19 +124,16 @@@ mergTimeVarName = 'time' mergLatVarName = 'latitude' mergLonVarName = 'longitude' - - filelistInstructions = dirname + '/*' - if filelist == None: + if filelist is None: filelist = glob.glob(filelistInstructions) - - #sat_img is the array that will contain all the masked frames + # sat_img is the array that will contain all the masked frames mergImgs = [] - #timelist of python time strings - timelist = [] + # timelist of python time strings + timelist = [] time2store = None - tempMaskedValueNp =[] - + tempMaskedValueNp = [] filelist.sort() nfiles = len(filelist) @@@ -271,20 -254,16 +271,21 @@@ def findCloudElements(mergImgs, timelis minCELonLimit = 0.0 maxCELatLimit = 0.0 maxCELonLimit = 0.0 - + nygrd = len(LAT[:, 0]); nxgrd = len(LON[0, :]) - - #openfile for storing ALL cloudElement information - cloudElementsFile = open((MAINDIRECTORY+'/textFiles/cloudElements.txt'),'wb') - #openfile for storing cloudElement information meeting user criteria i.e. MCCs in this case - cloudElementsUserFile = open((MAINDIRECTORY+'/textFiles/cloudElementsUserFile.txt'),'w') - - #NB in the TRMM files the info is hours since the time thus 00Z file has in 01, 02 and 03 times - for t in xrange(mergImgs.shape[0]): + + # openfile for storing ALL cloudElement information + cloudElementsFile = open( + (MAINDIRECTORY + '/textFiles/cloudElements.txt'), 'wb') + # openfile for storing cloudElement information meeting user criteria i.e. + # MCCs in this case + cloudElementsUserFile = open( + (MAINDIRECTORY + '/textFiles/cloudElementsUserFile.txt'), 'w') + + # NB in the TRMM files the info is hours since the time thus 00Z file has + # in 01, 02 and 03 times + for t in range(mergImgs.shape[0]): ++ #------------------------------------------------- # #textfile name for saving the data for arcgis # thisFileName = MAINDIRECTORY+'/' + (str(timelist[t])).replace(" ", "_") + '.txt' @@@ -460,60 -409,48 +461,59 @@@ currNetCDFTRMMData.conventions = 'COARDS' # dimensions currNetCDFTRMMData.createDimension('time', None) - currNetCDFTRMMData.createDimension('lat', len(LAT[:,0])) - currNetCDFTRMMData.createDimension('lon', len(LON[0,:])) - + currNetCDFTRMMData.createDimension('lat', len(LAT[:, 0])) + currNetCDFTRMMData.createDimension('lon', len(LON[0, :])) + # variables - TRMMprecip = ('time','lat', 'lon',) - times = currNetCDFTRMMData.createVariable('time', 'f8', ('time',)) - times.units = 'hours since '+ str(timelist[t])[:-6] - latitude = currNetCDFTRMMData.createVariable('latitude', 'f8', ('lat',)) - longitude = currNetCDFTRMMData.createVariable('longitude', 'f8', ('lon',)) - rainFallacc = currNetCDFTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip ) + TRMMprecip = ('time', 'lat', 'lon',) + times = currNetCDFTRMMData.createVariable( + 'time', 'f8', ('time',)) + times.units = 'hours since ' + str(timelist[t])[:-6] + latitude = currNetCDFTRMMData.createVariable( + 'latitude', 'f8', ('lat',)) + longitude = currNetCDFTRMMData.createVariable( + 'longitude', 'f8', ('lon',)) + rainFallacc = currNetCDFTRMMData.createVariable( + 'precipitation_Accumulation', 'f8', TRMMprecip) rainFallacc.units = 'mm' - longitude[:] = LON[0,:] - longitude.units = "degrees_east" - longitude.long_name = "Longitude" + longitude[:] = LON[0, :] + longitude.units = "degrees_east" + longitude.long_name = "Longitude" - latitude[:] = LAT[:,0] + latitude[:] = LAT[:, 0] latitude.units = "degrees_north" - latitude.long_name ="Latitude" + latitude.long_name = "Latitude" finalCETRMMvalues = ma.zeros((brightnesstemp.shape)) - #-----------End most of NETCDF file stuff ------------------------------------ + #-----------End most of NETCDF file stuff ----------------- - #populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon - #TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this + # populate cloudElementLatLons by unpacking the original values from loc to get the actual value for lat and lon + # TODO: KDW - too dirty... play with itertools.izip or zip and the enumerate with this # as cloudElement is masked - for index,value in np.ndenumerate(cloudElement): - if value != 0 : - lat_index,lon_index = index - lat_lon_tuple = (cloudElementLat[lat_index], cloudElementLon[lon_index],value) - - #generate the comma separated file for GIS + for index, value in np.ndenumerate(cloudElement): + if value != 0: + lat_index, lon_index = index + lat_lon_tuple = ( + cloudElementLat[lat_index], + cloudElementLon[lon_index], + value) + + # generate the comma separated file for GIS cloudElementLatLons.append(lat_lon_tuple) - #temp data for CE NETCDF file - brightnesstemp1[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = value - - if TRMMdirName: - finalCETRMMvalues[0,int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] = regriddedTRMM[int(np.where(LAT[:,0]==cloudElementLat[lat_index])[0]),int(np.where(LON[0,:]==cloudElementLon[lon_index])[0])] - CETRMMList.append((cloudElementLat[lat_index], cloudElementLon[lon_index], finalCETRMMvalues[0,cloudElementLat[lat_index], cloudElementLon[lon_index]])) - + # temp data for CE NETCDF file + brightnesstemp1[0, int(np.where(LAT[:, 0] == cloudElementLat[lat_index])[0]), int( + np.where(LON[0, :] == cloudElementLon[lon_index])[0])] = value + if TRMMdirName: + finalCETRMMvalues[0, int(np.where(LAT[:, 0] == cloudElementLat[lat_index])[0]), int(np.where(LON[0, :] == cloudElementLon[lon_index])[ + 0])] = regriddedTRMM[int(np.where(LAT[:, 0] == cloudElementLat[lat_index])[0]), int(np.where(LON[0, :] == cloudElementLon[lon_index])[0])] + CETRMMList.append((cloudElementLat[lat_index], + cloudElementLon[lon_index], + finalCETRMMvalues[0, + cloudElementLat[lat_index], + cloudElementLon[lon_index]])) - brightnesstemp[:] = brightnesstemp1[:] currNetCDFCEData.close() @@@ -917,54 -746,51 +917,53 @@@ def findPrecipRate(TRMMdirName, timelis minCEprecipRate = 0.0 try: - maxCEprecipRate = np.max(finalCETRMMvalues[np.nonzero(finalCETRMMvalues)]) - except: + maxCEprecipRate = np.max( + finalCETRMMvalues[np.nonzero(finalCETRMMvalues)]) + except BaseException: maxCEprecipRate = 0.0 - #add info to CLOUDELEMENTSGRAPH - #TODO try block + # add info to CLOUDELEMENTSGRAPH + # TODO try block for eachdict in CLOUD_ELEMENT_GRAPH.nodes(CEuniqueID): if eachdict[1]['uniqueID'] == CEuniqueID: - if not 'cloudElementPrecipTotal' in eachdict[1].keys(): + if 'cloudElementPrecipTotal' not in list(eachdict[1].keys()): eachdict[1]['cloudElementPrecipTotal'] = precipTotal - if not 'cloudElementLatLonTRMM' in eachdict[1].keys(): + if 'cloudElementLatLonTRMM' not in list(eachdict[1].keys()): eachdict[1]['cloudElementLatLonTRMM'] = finalCETRMMvalues - if not 'TRMMArea' in eachdict[1].keys(): + if 'TRMMArea' not in list(eachdict[1].keys()): eachdict[1]['TRMMArea'] = TRMMArea - if not 'CETRMMmin' in eachdict[1].keys(): + if 'CETRMMmin' not in list(eachdict[1].keys()): eachdict[1]['CETRMMmin'] = minCEprecipRate - if not 'CETRMMmax' in eachdict[1].keys(): + if 'CETRMMmax' not in list(eachdict[1].keys()): eachdict[1]['CETRMMmax'] = maxCEprecipRate - #clean up + # clean up precipTotal = 0.0 - latsrawTRMMData =[] + latsrawTRMMData = [] lonsrawTRMMData = [] - latsrawCloudElements=[] - lonsrawCloudElements=[] - finalCETRMMvalues =[] - CEprecipRate =[] - brightnesstemp =[] - TRMMdataDict ={} + latsrawCloudElements = [] + lonsrawCloudElements = [] + finalCETRMMvalues = [] + CEprecipRate = [] + brightnesstemp = [] + TRMMdataDict = {} return allCEnodesTRMMdata -#****************************************************************** +#****************************************************************** + + def findCloudClusters(CEGraph): ''' - Purpose:: - Determines the cloud clusters properties from the subgraphs in - the graph i.e. prunes the graph according to the minimum depth + Purpose:: + Determines the cloud clusters properties from the subgraphs in + the graph i.e. prunes the graph according to the minimum depth - Input:: - CEGraph: a Networkx directed graph of the CEs with weighted edges - according the area overlap between nodes (CEs) of consectuive frames - - Output:: - PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs + Input:: + CEGraph: a Networkx directed graph of the CEs with weighted edges + according the area overlap between nodes (CEs) of consectuive frames + Output:: + PRUNED_GRAPH: a Networkx directed graph of with CCs/ MCSs - ''' seenNode = [] @@@ -1112,148 -924,105 +1111,147 @@@ def findMCC(prunedGraph) MCCList = checkedNodesMCC(prunedGraph, treeTraversalList) for aDict in MCCList: for eachNode in aDict["fullMCSMCC"]: - addNodeMCSIdentifier(eachNode[0],eachNode[1]) - - #do check for if MCCs overlap + addNodeMCSIdentifier(eachNode[0], eachNode[1]) + + # do check for if MCCs overlap if MCCList: if len(MCCList) > 1: - for count in range(len(MCCList)): #for eachDict in MCCList: - #if there are more than two lists + # for eachDict in MCCList: + for count in range(len(MCCList)): + # if there are more than two lists if count >= 1: - #and the first node in this list - eachList = list(x[0] for x in MCCList[count]["possMCCList"]) - eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0])) + # and the first node in this list + eachList = list( + x[0] for x in MCCList[count]["possMCCList"]) + eachList.sort(key=lambda nodeID: ( + len(nodeID.split('C')[0]), nodeID.split('C')[0])) if eachList: fNode = eachList[0] - #get the lastNode in the previous possMCC list - eachList = list(x[0] for x in MCCList[(count-1)]["possMCCList"]) - eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0])) + # get the lastNode in the previous possMCC list + eachList = list( + x[0] for x in MCCList[(count - 1)]["possMCCList"]) + eachList.sort(key=lambda nodeID: ( + len(nodeID.split('C')[0]), nodeID.split('C')[0])) if eachList: lNode = eachList[-1] - if lNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode): - for aNode in CLOUD_ELEMENT_GRAPH.predecessors(fNode): + if lNode in CLOUD_ELEMENT_GRAPH.predecessors( + fNode): + for aNode in CLOUD_ELEMENT_GRAPH.predecessors( + fNode): if aNode in eachList and aNode == lNode: - #if edge_data is equal or less than to the exisitng edge in the tree append one to the other - if CLOUD_ELEMENT_GRAPH.get_edge_data(aNode,fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data(lNode,fNode)['weight']: - MCCList[count-1]["possMCCList"].extend(MCCList[count]["possMCCList"]) - MCCList[count-1]["fullMCSMCC"].extend(MCCList[count]["fullMCSMCC"]) - MCCList[count-1]["durationAandB"] += MCCList[count]["durationAandB"] - MCCList[count-1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"] - MCCList[count-1]["highestMCCnode"] = MCCList[count]["highestMCCnode"] - MCCList[count-1]["frameNum"] = MCCList[count]["frameNum"] + # if + # edge_data + # is + # equal + # or + # less + # than + # to + # the + # exisitng + # edge + # in + # the + # tree + # append + # one + # to + # the + # other + if CLOUD_ELEMENT_GRAPH.get_edge_data( + aNode, fNode)['weight'] <= CLOUD_ELEMENT_GRAPH.get_edge_data( + lNode, fNode)['weight']: + MCCList[count - 1]["possMCCList"].extend( + MCCList[count]["possMCCList"]) + MCCList[count - 1]["fullMCSMCC"].extend( + MCCList[count]["fullMCSMCC"]) + MCCList[count - + 1]["durationAandB"] += MCCList[count]["durationAandB"] + MCCList[count - + 1]["CounterCriteriaA"] += MCCList[count]["CounterCriteriaA"] + MCCList[count - + 1]["highestMCCnode"] = MCCList[count]["highestMCCnode"] + MCCList[count - + 1]["frameNum"] = MCCList[count]["frameNum"] removeList.append(count) - #update the MCCList + # update the MCCList if removeList: for i in removeList: - if (len(MCCList)-1) > i: + if (len(MCCList) - 1) > i: del MCCList[i] - removeList =[] - - #check if the nodes also meet the duration criteria and the shape crieria + removeList = [] + + # check if the nodes also meet the duration criteria and the shape + # crieria for eachDict in MCCList: - #order the fullMCSMCC list, then run maximum extent and eccentricity criteria - if (eachDict["durationAandB"] * TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] * TRES) <= MAXIMUM_DURATION: + # order the fullMCSMCC list, then run maximum extent and + # eccentricity criteria + if (eachDict["durationAandB"] * + TRES) >= MINIMUM_DURATION and (eachDict["durationAandB"] * + TRES) <= MAXIMUM_DURATION: eachList = list(x[0] for x in eachDict["fullMCSMCC"]) - eachList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0])) + eachList.sort(key=lambda nodeID: ( + len(nodeID.split('C')[0]), nodeID.split('C')[0])) eachMCCList = list(x[0] for x in eachDict["possMCCList"]) - eachMCCList.sort(key=lambda nodeID:(len(nodeID.split('C')[0]), nodeID.split('C')[0])) - - #update the nodemcsidentifer behavior - #find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I', - #find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D' - #ensure that everything between is listed as 'M' - for eachNode in eachList[:(eachList.index(eachMCCList[0]))]: - addNodeMCSIdentifier(eachNode,'I') - - addNodeMCSIdentifier(eachMCCList[0],'M') - - for eachNode in eachList[(eachList.index(eachMCCList[-1])+1):]: + eachMCCList.sort(key=lambda nodeID: ( + len(nodeID.split('C')[0]), nodeID.split('C')[0])) + + # update the nodemcsidentifer behavior + # find the first element eachMCCList in eachList, and ensure everything ahead of it is indicated as 'I', + # find last element in eachMCCList in eachList and ensure everything after it is indicated as 'D' + # ensure that everything between is listed as 'M' + for eachNode in eachList[:( + eachList.index(eachMCCList[0]))]: + addNodeMCSIdentifier(eachNode, 'I') + + addNodeMCSIdentifier(eachMCCList[0], 'M') + + for eachNode in eachList[( + eachList.index(eachMCCList[-1]) + 1):]: addNodeMCSIdentifier(eachNode, 'D') - #update definiteMCS list - for eachNode in orderedPath[(orderedPath.index(eachMCCList[-1])+1):]: + # update definiteMCS list + for eachNode in orderedPath[( + orderedPath.index(eachMCCList[-1]) + 1):]: addNodeMCSIdentifier(eachNode, 'D') - #run maximum extent and eccentricity criteria - maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity(eachList) - #print "maxExtentNode, definiteMCCFlag ", maxExtentNode, definiteMCCFlag - if definiteMCCFlag == True: + # run maximum extent and eccentricity criteria + maxExtentNode, definiteMCCFlag = maxExtentAndEccentricity( + eachList) + if definiteMCCFlag: definiteMCC.append(eachList) - definiteMCS.append(orderedPath) - - #reset for next subGraph + + # reset for next subGraph aSubGraph.clear() - orderedPath=[] - MCCList =[] - MCSList =[] + orderedPath = [] + MCCList = [] + MCSList = [] definiteMCSFlag = False - + return definiteMCC, definiteMCS #****************************************************************** -def traverseTree(subGraph,node, stack, checkedNodes=None): + + +def traverseTree(subGraph, node, stack, checkedNodes=None): ''' - Purpose:: - To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm + Purpose:: + To traverse a tree using a modified depth-first iterative deepening (DFID) search algorithm + + Input:: + subGraph: a Networkx DiGraph representing a CC + lengthOfsubGraph: an integer representing the length of the subgraph + node: a string representing the node currently being checked + stack: a list of strings representing a list of nodes in a stack functionality + i.e. Last-In-First-Out (LIFO) for sorting the information from each visited node + checkedNodes: a list of strings representing the list of the nodes in the traversal - Input:: - subGraph: a Networkx DiGraph representing a CC - lengthOfsubGraph: an integer representing the length of the subgraph - node: a string representing the node currently being checked - stack: a list of strings representing a list of nodes in a stack functionality - i.e. Last-In-First-Out (LIFO) for sorting the information from each visited node + Output:: checkedNodes: a list of strings representing the list of the nodes in the traversal - - Output:: - checkedNodes: a list of strings representing the list of the nodes in the traversal - Assumptions: - frames are ordered and are equally distributed in time e.g. hrly satellite images - + Assumptions: + frames are ordered and are equally distributed in time e.g. hrly satellite images - ''' if len(checkedNodes) == len(subGraph): return checkedNodes @@@ -1270,10 -1038,10 +1268,9 @@@ if parent not in checkedNodes and parent not in stack: for child in downOneLevel: if child not in checkedNodes and child not in stack: - stack.insert(0,child) - - stack.insert(0,parent) + stack.insert(0, child) + stack.insert(0, parent) - for child in downOneLevel: if child not in checkedNodes and child not in stack: if len(subGraph.predecessors(child)) > 1 or node in checkedNodes: @@@ -1556,62 -1231,38 +1553,63 @@@ def updateMCCList potentialMCCList[index]["highestMCCnode"] = node return potentialMCCList - #if this frameNum doesn't exist and this frameNum is less than the MCC node max frame Num (including 0), then append to fullMCSMCC list - if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"]==0: + # if this frameNum doesn't exist and this frameNum is less than + # the MCC node max frame Num (including 0), then append to + # fullMCSMCC list + if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0: stage = 'I' - if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False: - potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 1, "durationAandB": 0, "highestMCCnode":"", "frameNum":0}) + if CounterCriteriaAFlag and CounterCriteriaBFlag == False: + potentialMCCList.append( + { + "possMCCList": [], + "fullMCSMCC": [ + (node, + stage)], + "CounterCriteriaA": 1, + "durationAandB": 0, + "highestMCCnode": "", + "frameNum": 0}) return potentialMCCList elif CounterCriteriaAFlag == False and CounterCriteriaBFlag == False: - potentialMCCList.append({"possMCCList":[], "fullMCSMCC":[(node,stage)], "CounterCriteriaA": 0, "durationAandB": 0, "highestMCCnode":"", "frameNum":0}) + potentialMCCList.append( + { + "possMCCList": [], + "fullMCSMCC": [ + (node, + stage)], + "CounterCriteriaA": 0, + "durationAandB": 0, + "highestMCCnode": "", + "frameNum": 0}) return potentialMCCList - #if predecessor and this frame number already exist in the MCC list, add the current node to the fullMCSMCC list - if existingFrameFlag == True: - if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False: - potentialMCCList[index]["fullMCSMCC"].append((node,stage)) - potentialMCCList[index]["CounterCriteriaA"] +=1 + # if predecessor and this frame number already exist in the MCC + # list, add the current node to the fullMCSMCC list + if existingFrameFlag: + if CounterCriteriaAFlag and CounterCriteriaBFlag == False: + potentialMCCList[index]["fullMCSMCC"].append((node, stage)) + potentialMCCList[index]["CounterCriteriaA"] += 1 return potentialMCCList - if CounterCriteriaAFlag == False: - potentialMCCList[index]["fullMCSMCC"].append((node,stage)) - return potentialMCCList - - if predecessorsFlag == False: - successorsFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2) - - if successorsFlag == True: - for eachNode in potentialMCCList[index]["possMCCList"]: - if int((eachNode[0].split('CE')[0]).split('F')[1]) == frameNum: + if not CounterCriteriaAFlag: + potentialMCCList[index]["fullMCSMCC"].append((node, stage)) + return potentialMCCList + + if not predecessorsFlag: + successorsFlag, index = isThereALink( + prunedGraph, 2, node, potentialMCCList, 2) + + if successorsFlag: + for eachNode in potentialMCCList[index]["possMCCList"]: + if int( + (eachNode[0].split('CE')[0]).split('F')[1]) == frameNum: existingFrameFlag = True - - if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True: + + if CounterCriteriaAFlag and CounterCriteriaBFlag: stage = 'M' - potentialMCCList[index]["possMCCList"].append((node,stage)) - potentialMCCList[index]["fullMCSMCC"].append((node,stage)) + potentialMCCList[index]["possMCCList"].append( + (node, stage)) + potentialMCCList[index]["fullMCSMCC"].append((node, stage)) ++ if frameNum > potentialMCCList[index]["frameNum"] or potentialMCCList[index]["frameNum"] == 0: potentialMCCList[index]["frameNum"] = frameNum potentialMCCList[index]["highestMCCnode"] = node @@@ -1660,24 -1305,19 +1658,25 @@@ return potentialMCCList if potentialMCCList[index]["frameNum"] == 0 or frameNum <= potentialMCCList[index]["frameNum"]: - if CounterCriteriaAFlag == True and CounterCriteriaBFlag == False: - potentialMCCList[index]["fullMCSMCC"].append((node,stage)) - potentialMCCList[index]["CounterCriteriaA"] +=1 + if CounterCriteriaAFlag and CounterCriteriaBFlag == False: + potentialMCCList[index]["fullMCSMCC"].append( + (node, stage)) + potentialMCCList[index]["CounterCriteriaA"] += 1 return potentialMCCList elif CounterCriteriaAFlag == False: - potentialMCCList[index]["fullMCSMCC"].append((node,stage)) + potentialMCCList[index]["fullMCSMCC"].append( + (node, stage)) return potentialMCCList else: - successorsMCSFlag, index = isThereALink(prunedGraph, 2,node,potentialMCCList,2) - if successorsMCSFlag == True: - if CounterCriteriaAFlag == True and CounterCriteriaBFlag == True: - potentialMCCList[index]["possMCCList"].append((node,'M')) - potentialMCCList[index]["fullMCSMCC"].append((node,'M')) + successorsMCSFlag, index = isThereALink( + prunedGraph, 2, node, potentialMCCList, 2) + if successorsMCSFlag: + if CounterCriteriaAFlag and CounterCriteriaBFlag: + potentialMCCList[index]["possMCCList"].append( + (node, 'M')) + potentialMCCList[index]["fullMCSMCC"].append( + (node, 'M')) ++ potentialMCCList[index]["durationAandB"] += 1 if frameNum > potentialMCCList[index]["frameNum"]: potentialMCCList[index]["frameNum"] = frameNum @@@ -2071,24 -1641,22 +2070,23 @@@ def allDescendants(path, aNode) path = path + aNode #i.e. PRUNED_GRAPH.predecessors(aNode) is empty return path, numOfChildren - except: - #i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception + # i.e. PRUNED_GRAPH.predecessors(aNode) threw an exception + except BaseException: return path, numOfChildren #****************************************************************** -def addInfothisDict (thisNode, cloudElementArea,criteriaB): - ''' - Purpose:: - Update original dictionary node with information - Input:: - thisNode: a string representing the unique ID of a node - cloudElementArea: a floating-point number representing the area of the cloud element - criteriaB: a masked array of floating-point numbers representing the lat,lons meeting the criteria - Output:: None +def addInfothisDict(thisNode, cloudElementArea, criteriaB): + ''' + Purpose:: + Update original dictionary node with information + + Input:: + thisNode: a string representing the unique ID of a node + cloudElementArea: a floating-point number representing the area of the cloud element + criteriaB: a masked array of floating-point numbers representing the lat,lons meeting the criteria + Output:: None - ''' for eachdict in CLOUD_ELEMENT_GRAPH.nodes(thisNode): if eachdict[1]['uniqueID'] == thisNode: @@@ -3323,29 -2737,24 +3321,28 @@@ def displaySize(finalMCCList) ax.set_title(title) ax.fmt_xdata = mdates.DateFormatter('%Y-%m-%d%H:%M:%S') fig.autofmt_xdate() - - plt.subplots_adjust(bottom=0.2) - - imgFilename = MAINDIRECTORY+'/images/'+ str(count)+'MCS.gif' - plt.savefig(imgFilename, facecolor=fig.get_facecolor(), transparent=True) - - #if time in not already in the time list, append it - timeList=[] + + imgFilename = MAINDIRECTORY + '/images/' + str(count) + 'MCS.gif' + plt.savefig( + imgFilename, + facecolor=fig.get_facecolor(), + transparent=True) + + # if time in not already in the time list, append it + timeList = [] count += 1 - return + return #****************************************************************** -def displayPrecip(finalMCCList): + + +def displayPrecip(finalMCCList): ''' - Purpose:: - To create a figure showing the precip rate verse time for each MCS + Purpose:: + To create a figure showing the precip rate verse time for each MCS - Input:: - finalMCCList: a list of dictionaries representing a list of nodes representing a MCC + Input:: + finalMCCList: a list of dictionaries representing a list of nodes representing a MCC Output:: None @@@ -3730,25 -3093,24 +3727,24 @@@ def plotAccuInTimeRange(starttime, endt accuTRMMData.createDimension('time', None) accuTRMMData.createDimension('lat', nygrdTRMM) accuTRMMData.createDimension('lon', nxgrdTRMM) - + # variables - TRMMprecip = ('time','lat', 'lon',) + TRMMprecip = ('time', 'lat', 'lon',) times = accuTRMMData.createVariable('time', 'f8', ('time',)) - times.units = 'hours since '+ starttime[:-6] + times.units = 'hours since ' + starttime[:-6] latitude = accuTRMMData.createVariable('latitude', 'f8', ('lat',)) longitude = accuTRMMData.createVariable('longitude', 'f8', ('lon',)) - rainFallacc = accuTRMMData.createVariable('precipitation_Accumulation', 'f8',TRMMprecip) + rainFallacc = accuTRMMData.createVariable( + 'precipitation_Accumulation', 'f8', TRMMprecip) rainFallacc.units = 'mm' - longitude[:] = LONTRMM[0,:] - longitude.units = "degrees_east" - longitude.long_name = "Longitude" + longitude[:] = LONTRMM[0, :] + longitude.units = "degrees_east" + longitude.long_name = "Longitude" - latitude[:] = LATTRMM[:,0] + latitude[:] = LATTRMM[:, 0] latitude.units = "degrees_north" - latitude.long_name ="Latitude" - + latitude.long_name = "Latitude" - rainFallacc[:] = accuPrecipRate[:] accuTRMMData.close() @@@ -4235,22 -3515,18 +4231,23 @@@ def createTextFile(finalMCCList, identi #****************************************************************** # PLOTTING UTIL SCRIPTS #****************************************************************** -def to_percent(y,position): + + +def to_percent(y, position): ''' - Purpose:: - Utility script for generating the y-axis for plots + Purpose:: + Utility script for generating the y-axis for plots ''' - return (str(100*y)+'%') + return (str(100 * y) + '%') #****************************************************************** + + def colorbar_index(ncolors, nlabels, cmap): ''' - Purpose:: - Utility script for crating a colorbar - Taken from http://stackoverflow.com/questions/18704353/correcting-matplotlib-colorbar-ticks + Purpose:: + Utility script for crating a colorbar + Taken from http://stackoverflow.com/questions/18704353/correcting-matplotlib-colorbar-ticks ++ ''' cmap = cmap_discretize(cmap, ncolors) mappable = cm.ScalarMappable(cmap=cmap) @@@ -4293,22 -3567,22 +4290,19 @@@ def cmap_discretize(cmap, N) # def preprocessingMERG(MERGdirname): # ''' # Purpose:: -# Utility script for unzipping and converting the merg*.Z files from Mirador to +# Utility script for unzipping and converting the merg*.Z files from Mirador to # NETCDF format. The files end up in a folder called mergNETCDF in the directory # where the raw MERG data is -# NOTE: VERY RAW AND DIRTY - +# NOTE: VERY RAW AND DIRTY - # Input:: # Directory to the location of the raw MERG files, preferably zipped - - # Output:: # none -- # Assumptions:: # 1 GrADS (http://www.iges.org/grads/gadoc/) and lats4D (http://opengrads.org/doc/scripts/lats4d/) -# have been installed on the system and the user can access +# have been installed on the system and the user can access # 2 User can write files in location where script is being called -# 3 the files havent been unzipped +# 3 the files havent been unzipped # ''' # os.chdir((MERGdirname+'/')) http://git-wip-us.apache.org/repos/asf/climate/blob/51d9dce1/mccsearch/code/mccSearchUI.py ---------------------------------------------------------------------- diff --cc mccsearch/code/mccSearchUI.py index 36fe642,a227143..feea1d9 --- a/mccsearch/code/mccSearchUI.py +++ b/mccsearch/code/mccSearchUI.py @@@ -18,10 -18,10 +18,11 @@@ # Wizard for running the mccSearch program ''' + import os import networkx as nx -#mccSearch modules -import mccSearch +# mccSearch modules +from mccSearch import * + def main(): CEGraph = nx.DiGraph()
