-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsplit_by_concavity.py
223 lines (166 loc) · 8.54 KB
/
split_by_concavity.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
#matplotlib pyplot hist2D
#windows concavity catcher test
import pandas as pd
import csv
import os
import sys
#LSDTopoTools specific imports
#Loading the LSDTT setup configuration
setup_file = open('chi_automation.config','r')
LSDMT_PT = setup_file.readline().rstrip()
LSDMT_MF = setup_file.readline().rstrip()
Iguanodon = setup_file.readline().rstrip()
setup_file.close()
sys.path.append(LSDMT_PT)
sys.path.append(LSDMT_MF)
sys.path.append(Iguanodon)
from LSDPlottingTools import LSDMap_MOverNPlotting as MN
from LSDMapFigure import PlottingHelpers as Helper
#target = os.path.join('R:\\','LSDTopoTools','Topographic_projects','full_himalaya')
target = '/exports/csce/datastore/geos/users/s1134744/LSDTopoTools/Topographic_projects/full_himalaya_5000/'
#output = os.path.join('C:\\output2\\')
remove_GLIMS = False
def writeHeader(file_name,target_name):
with open(file_name,'r') as sourceheader_csv:
pandasDF=pd.read_csv(sourceheader_csv,delimiter=',')
header_list = pandasDF.columns.values.tolist()
with open(target_name,'wb') as writeheader_csv:
csvWriter = csv.writer(writeheader_csv,delimiter = ',')
csvWriter.writerow(header_list)
def pathCollector(path,name):
#returns lists of paths and names
path = os.path.join(path,name+'.csv')
with open(path) as csvfile:
csvReader = csv.reader(csvfile,delimiter=',')
next(csvReader)
full_paths = []
dem_names = []
write_names = []
for row in csvReader:
max_basin = (int(row[6])/2)+int(row[5])
part_1 = str(row[0])
part_1 = part_1.replace('.','_')
part_2 = str(("%.2f" %float(row[2])))+'_'+str(("%.2f" %float(row[3])))
part_2 = part_2.replace('.','_')
full_path = os.path.join(target,part_1,part_2+'_'+part_1+'_'+str(row[1]),str(row[5]))
dem_name = part_1+'_'+str(row[1])
write_name = str(row[1])+str(row[5])+'_'+str((int(row[6])/2)+int(row[5]))
full_paths.append(full_path)
dem_names.append(dem_name)
write_names.append(write_name)
return full_paths,dem_names,write_names
def concavityCatcher(full_path,write_name,processed=False):
#returns the basin_key and median concavity
write_name = '/'+write_name
#reading in the basin info
BasinDF = Helper.ReadMCPointsCSV(full_path,write_name)
#Getting mn data
PointsDF = MN.GetMOverNRangeMCPoints(BasinDF,start_movern=0.1,d_movern=0.05,n_movern=18)
#extract basin key and concavity as list
basin_series = PointsDF["basin_key"]
concavity_series = PointsDF["Median_MOverNs"]
basin_key = basin_series.tolist()
basin_keys = []
for x in basin_key:
x = int(x)
basin_keys.append(x)
concavities = concavity_series.tolist()
if not processed:
return basin_keys,concavities
if processed:
#processed_concavities = []
#print("this is the processed DF")
#print processedDF
processed_basin = processedDF["basin_key"]
processed_concavity = processedDF["Median_MOverNs"]
basin_list = processed_basin.tolist()
concavity_list = processed_concavity.tolist()
return basin_list,concavity_list
def getBasinLatLon(full_path,write_name):
#print "opened function"
#print full_path+write_name+'_AllBasinsInfo.csv'
with open(full_path+'/'+write_name+'_AllBasinsInfo.csv','r') as basinInfo:
#part of glacier removal
basinDF = pd.read_csv(basinInfo,delimiter=',')
lat = basinDF['outlet_latitude']
lon = basinDF['outlet_longitude']
lat_list = lat.tolist()
lon_list = lon.tolist()
#print lat_list,lon_list
return lat_list,lon_list
def getDisorderConcavity(full_path,write_name):
with open(full_path+'/'+write_name+'_fullstats_disorder_uncert.csv','r') as disorderInfo:
#part of glacier removal
disorderDF = pd.read_csv(disorderInfo,delimiter=',')
disorderConcavity = disorderDF[' best_fit_for_all_tribs']
disorder_list = disorderConcavity.tolist()
corrected_disorder = []
for x in disorder_list:
correct = "%.2f" %float(x)
corrected_disorder.append(correct)
return corrected_disorder
def countConcavity(dataFrame,concavity):
concavityFrame = dataFrame[dataFrame['concavity_bootstrap'] == concavity]
concavitySeries = concavityFrame['concavity_bootstrap']
to_list = concavitySeries.tolist()
disorder_concavity = "%.2f" %float(concavity)
disorderFrame = dataFrame[dataFrame['concavity_disorder'] == concavity]
disorderSeries = disorderFrame['concavity_disorder']
disorder_list = disorderSeries.tolist()
with open(target+'/'+'concavity_bootstrap_summary.csv','a') as csvwrite:
csvWriter = csv.writer(csvwrite,delimiter=',')
csvWriter.writerow((concavity,len(to_list),len(disorder_list)))
print "There are bootstrap %s, disorder %s basins with a concavity of %s"%(len(to_list),len(disorder_list),concavity)
def ksnCatcher(full_path,dem_name,basin_key,concavity):
#returns dataframe with mchi(ksn) for each basin based on the correct concavity
try:
with open(full_path+'/'+dem_name+'0_3_MChiSegmented_burned.csv','r') as mChicsv:
mchiPandas = pd.read_csv(mChicsv,delimiter=',')
selectedDF = mchiPandas.loc[mchiPandas['basin_key'] == int(basin_key)]
print "testing %s %s %s"%(full_path,dem_name,basin_key)
return selectedDF
except Exception as e:
print e
print("Error, fault in KSN catcher, this tile is probably missing %s %s\n"%(full_path,dem_name))
print full_path+dem_name+'_MChiSegmented_burned.csv'
x_i = 0
names = ['himalaya_processed','himalaya_b_processed','himalaya_c_processed']
for name in names:
full_paths,dem_names,write_names = pathCollector(target,name)
#testing to see if output files exist:
#full_paths = ['/exports/csce/datastore/geos/users/s1134744/LSDTopoTools/Topographic_projects/full_himalaya/himalaya_27_5/27_50_88_20_himalaya_27_5_14/20000/']
#dem_names = ['himalaya_27_5_14']
#write_names = ['1420000_35000']
m_n_list = [0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,0.7,0.75,0.8,0.85,0.9,0.95]
#m_n_list = [0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6]
for c in m_n_list:
c = str(c)
c = c.replace('.','_')
for d,e in zip(full_paths,dem_names):
if not os.path.isfile(target+'/'+'raw/'+c+'_0.3_MChiSegmented_burned.csv'):
try:
writeHeader(file_name=d+'/'+e+c+'_MChiSegmented_burned.csv',target_name=target+'raw/'+c+'_0.3_MChiSegmented_burned.csv')
except:
print("source for headers not found, looping through lists until one is.",d+'/'+e+c+'_MChiSegmented_burned.csv')
for x,y,z in zip(full_paths,dem_names,write_names):
try:
basin_keys,concavities = concavityCatcher(x,z)
#testing length of strings provides a basic error control
if len(basin_keys) == len(concavities):
print("got basin key list and concavity list, lengths match so going ahead and collecting corresponding ksn data")
for a,b in zip(basin_keys,concavities):
b = str(b)
b = b.replace('.','_')
ksnDF = ksnCatcher(x,y,a,b)
try:
ksnDF.to_csv(target+'raw/'+b+'_0.3_MChiSegmented_burned.csv',mode='a',header=False,index=False)
print("saving to...",target+'raw/'+b+'_0.3_MChiSegmented_burned.csv')
print("got data for %s %s %s"%(y,a,b))
except:
print("ERROR: problem exporting dataframe to csv at %s %s %s"%(y,a,b))
else:
print("basin key/concavity strings are not an equal length")
print x_i
x_i+=1
except:
print("ERROR: Problem getting source concavity/basin data. Skipping tile... %s"%(y))