Skip to content

Commit caf65f9

Browse files
calumbradburycalumbradbury
committed
first commit
0 parents  commit caf65f9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+7274
-0
lines changed

‎backup_trmm_stats.py

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#backup stats writer
2+
#recalculates basin average and writes output to all_basins_trmmm
3+
import csv
4+
import pandas as pd
5+
import sys
6+
7+
target = '/exports/csce/datastore/geos/users/s1134744/LSDTopoTools/Topographic_projects/Himalayan_front/himalaya_all/'
8+
9+
#LSDTopoTools specific imports
10+
#Loading the LSDTT setup configuration
11+
setup_file = open('/exports/csce/datastore/geos/users/s1134744/LSDTopoTools/Git_projects/LSDAutomation/chi_analysis_automation/chi_automation.config','r')
12+
LSDMT_PT = setup_file.readline().rstrip()
13+
LSDMT_MF = setup_file.readline().rstrip()
14+
Iguanodon = setup_file.readline().rstrip()
15+
setup_file.close()
16+
17+
sys.path.append(LSDMT_PT)
18+
sys.path.append(LSDMT_MF)
19+
sys.path.append(Iguanodon)
20+
21+
from LSDPlottingTools import LSDMap_MOverNPlotting as MN
22+
from LSDMapFigure import PlottingHelpers as Helper
23+
import Iguanodon31 as Ig
24+
25+
#creating csv with rainfall averaged over basin segments, and including MN data
26+
with open(target+"_output_basin_TRMM_recalculated.csv","wb") as csvfile:
27+
csvWriter = csv.writer(csvfile,delimiter=',')
28+
csvWriter.writerow(("basin_key","mean jun/jul/aug rainfall (mm)"))
29+
30+
#opening source of climate data
31+
with open(target+'_output_MChiSegmented_nodata.csv','r') as csvfile:
32+
pandasDF = pd.read_csv(csvfile,delimiter=',')
33+
34+
#opening basin directory
35+
with open(target+'_output_AllBasinsInfo.csv','r') as csvfile_2:
36+
csvReader = csv.reader(csvfile_2,delimiter=',')
37+
next(csvReader)
38+
39+
for row in csvReader:
40+
basin_number = int(row[5])
41+
selected_DF = pandasDF.loc[pandasDF['basin_key'] == basin_number]
42+
#getting burned data series for the basin
43+
pandas_list = selected_DF['burned_data']
44+
#print pandas_list
45+
mean_rainfall = pandas_list.mean()
46+
47+
with open(target+'_output_basin_TRMM_recalculated.csv', 'a') as csvfile_3:
48+
csvWriter = csv.writer(csvfile_3,delimiter=',')
49+
csvWriter.writerow((basin_number,mean_rainfall))
50+
51+
with open(target+'_output_basin_TRMM.csv','r') as csvfile:
52+
pandas_a = pd.read_csv(csvfile,delimiter=',')
53+
with open(target+'_output_basin_TRMM_recalculated.csv','r') as csvfile_b:
54+
pandas_b = pd.read_csv(csvfile_b,delimiter=',')
55+
pandas_a = pandas_a.merge(pandas_b, on=["basin_key"])
56+
pandas_a.to_csv(target+'_output_basin_TRMM_new.csv', mode = "w", header = True, index = False)

‎basin_ks_testing.py

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#BASIN STATS INCLUDING KS
2+
3+
from scipy import stats
4+
import os
5+
import csv
6+
import pandas as pd
7+
8+
directory = '/exports/csce/datastore/geos/users/s1134744/LSDTopoTools/Topographic_projects/'
9+
10+
files = ['full_himalaya/full_concavity_basins_summary.csv','full_himalaya/concavity_bootstrap_basins_summary_processed.csv',
11+
'full_himalaya_5000/full_concavity_basins_summary.csv','full_himalaya_5000/concavity_bootstrap_basins_summary_processed.csv']
12+
13+
def returnSeries(source):
14+
DF = pd.read_csv(source,delimiter=',')
15+
series = DF['concavity_bootstrap']
16+
return series
17+
18+
def ks_2sample_test(pandasSeries_A,pandasSeries_B):
19+
list_A = pandasSeries_A.tolist()
20+
list_B = pandasSeries_B.tolist()
21+
statistic,p_value = stats.ks_2samp(list_A,list_B)
22+
return statistic,p_value
23+
24+
25+
full_full_DF = returnSeries(directory+files[0])
26+
full_GLIMS_DF = returnSeries(directory+files[1])
27+
clip_full_DF = returnSeries(directory+files[2])
28+
clip_GLIMS_DF = returnSeries(directory+files[3])
29+
30+
print "full DEM/full DEM",ks_2sample_test(full_full_DF,full_full_DF)
31+
print "full DEM/FULL-GLIMS",ks_2sample_test(full_full_DF,full_GLIMS_DF)
32+
print "full DEM/Clip-Full",ks_2sample_test(full_full_DF,clip_full_DF)
33+
print "full DEM/Clip-GLIMS",ks_2sample_test(full_full_DF,clip_GLIMS_DF)
34+
print ".............\n"
35+
print "full GLIMS/full DEM",ks_2sample_test(full_GLIMS_DF,full_full_DF)
36+
print "full GLIMS/FULL-GLIMS",ks_2sample_test(full_GLIMS_DF,full_GLIMS_DF)
37+
print "full GLIMS/Clip-Full",ks_2sample_test(full_GLIMS_DF,clip_full_DF)
38+
print "full GLIMS/Clip-GLIMS",ks_2sample_test(full_GLIMS_DF,clip_GLIMS_DF)
39+
print ".............\n"
40+
print "clip FULL/full DEM",ks_2sample_test(clip_full_DF,full_full_DF)
41+
print "clip FULL/FULL-GLIMS",ks_2sample_test(clip_full_DF,full_GLIMS_DF)
42+
print "clip FULL/Clip-Full",ks_2sample_test(clip_full_DF,clip_full_DF)
43+
print "clip FULL/Clip-GLIMS",ks_2sample_test(clip_full_DF,clip_GLIMS_DF)
44+
print ".............\n"
45+
print "clip GLIMS/full DEM",ks_2sample_test(clip_GLIMS_DF,full_full_DF)
46+
print "clip GLIMS/FULL-GLIMS",ks_2sample_test(clip_GLIMS_DF,full_GLIMS_DF)
47+
print "clip GLIMS/Clip-Full",ks_2sample_test(clip_GLIMS_DF,clip_full_DF)
48+
print "clip GLIMS/Clip-GLIMS",ks_2sample_test(clip_GLIMS_DF,clip_GLIMS_DF)

‎basin_precip_gradient.py

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#script to extract max/min precipitation from each basin. Used to calculate gradient
2+
import matplotlib
3+
4+
matplotlib.use("Agg")
5+
import pandas as pd
6+
import csv
7+
import os
8+
from matplotlib import pyplot as plt
9+
10+
target = '/exports/csce/datastore/geos/users/s1134744/LSDTopoTools/Topographic_projects/Himalayan_front/'
11+
#name = 'himalaya_processed.csv'
12+
13+
def getMaxMin(dataFrame,basin_key):
14+
basinDataFrame = dataFrame.loc[dataFrame["basin_key"] == basin_key]
15+
try:
16+
precipitationSeries = basinDataFrame["precipitation"]
17+
18+
except:
19+
precipitationSeries = basinDataFrame["secondary_burned_data"]
20+
#print precipitationSeries
21+
22+
#maxPrecip = precipitationSeries.max()
23+
#minPrecip = precipitationSeries.min()
24+
maxPrecip = precipitationSeries.first_valid_index()
25+
minPrecip = precipitationSeries.last_valid_index()
26+
27+
28+
return maxPrecip, minPrecip
29+
30+
31+
def pathCollector(path,name):
32+
#returns lists of paths and names
33+
with open(path+name+'.csv','r') as csvfile:
34+
csvReader = csv.reader(csvfile,delimiter=',')
35+
next(csvReader)
36+
full_paths = []
37+
dem_names = []
38+
write_names = []
39+
for row in csvReader:
40+
max_basin = (int(row[6])/2)+int(row[5])
41+
full_path = path+str(row[0])+'/'+("%.2f" %float(row[2]))+'_'+("%.2f" %float(row[3]))+'_'+str(row[0])+'_'+str(row[1])+'/'+str(row[5])+'/'
42+
dem_name = str(row[0])+'_'+str(row[1])
43+
write_name = str(row[1])+str(row[5])+'_'+str((int(row[6])/2)+int(row[5]))
44+
full_paths.append(full_path)
45+
dem_names.append(dem_name)
46+
write_names.append(write_name)
47+
return full_paths,dem_names,write_names
48+
49+
def basinLists(path):
50+
with open(path+'_AllBasinsInfo.csv','r') as basincsv:
51+
basinPandas = pd.read_csv(basincsv,delimiter=',')
52+
basins = basinPandas["basin_key"]
53+
basin_list = basins.tolist()
54+
return basin_list
55+
56+
def getMChiSegmentedPandas(path):
57+
with open(path+'_MChiSegmented_burned.csv') as mChiSource:
58+
pandasDF = pd.read_csv(mChiSource, delimiter=',')
59+
return pandasDF
60+
61+
def mainOperation(full_paths,dem_names,write_names,dem_record,basins,maxs,mins):
62+
63+
#looping through each tile
64+
for x,y,z in zip(full_paths,dem_names,write_names):
65+
#getting basin list for tile
66+
try:
67+
basin_list = basinLists(x+z)
68+
pandasDF = getMChiSegmentedPandas(x+z)
69+
for a in basin_list:
70+
max_precip,min_precip = getMaxMin(pandasDF,a)
71+
dem_record.append(y)
72+
basins.append(a)
73+
maxs.append(max_precip)
74+
mins.append(min_precip)
75+
except(IOError):
76+
print("IOError, some info does not exist %s"%(y))
77+
78+
return dem_record,basins,maxs,mins
79+
80+
def scatterPlot(dataFrame):
81+
82+
# Create a figure
83+
fig = plt.figure(1, figsize=(18,9))
84+
85+
# Create an axes
86+
ax = fig.add_subplot(111)
87+
#plt.ylabel("", fontsize = 24)
88+
plt.title(("precip_gradient"), fontsize = 32)
89+
90+
# Create the boxplot
91+
#bp = ax.boxplot(data_to_plot, labels=header_list, showfliers=False)
92+
bp = dataFrame.plot.scatter(x=[2],y=[3],c='DarkBlue')
93+
94+
plt.tick_params(axis='both', which='major', labelsize=18)
95+
# Save the figure
96+
fig.savefig(target+'precip_gradient_scatter.png', bbox_inches='tight')
97+
#required to clear the axes. Each call of this function wouldn't do that otherwise.
98+
plt.cla()
99+
100+
#getting base lists
101+
102+
full_paths,dem_names,write_names = pathCollector(target,'himalaya_processed')
103+
full_paths_b,dem_names_b,write_names_b = pathCollector(target,'himalaya_b_processed')
104+
105+
dem_record = []
106+
basins = []
107+
maxs = []
108+
mins = []
109+
110+
dem_record,basins,maxs,mins = mainOperation(full_paths,dem_names,write_names,dem_record,basins,maxs,mins)
111+
112+
dem_record,basins,maxs,mins = mainOperation(full_paths_b,dem_names_b,write_names_b,dem_record,basins,maxs,mins)
113+
114+
115+
116+
demDF = pd.Series(dem_record)
117+
basinDF = pd.Series(basins)
118+
maxDF = pd.Series(maxs)
119+
minDF = pd.Series(mins)
120+
121+
print demDF,basinDF,maxDF,minDF
122+
export_DF = pd.concat([demDF,basinDF,maxDF,minDF],axis=1)
123+
export_DF.to_csv(target+'precip_gradient_data_first_last.csv',mode='w',header=True,index=False)
124+
scatterPlot(export_DF)

0 commit comments

Comments
 (0)