-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathexperiments.py
155 lines (126 loc) · 6.19 KB
/
experiments.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
#!/usr/bin/python3
"""
This script launches experiments and save results.
Another script is used to agregate results.
"""
import sys
import os
import glob
import subprocess
from tqdm import tqdm, trange
import numpy as np
from datetime import timedelta
import csv
from collections import defaultdict
columnNames1 = ["Budget", "ExpectRemainingTime", "Deadline", "ExecutionTime"]
columnNames2 = ["ChoosingDuration", "NbResamplers", "NbDegradedNodes"]
columnNames3 = ["NbCycles", "NbDegradedCycles", "NbEdges"]
columnNames = columnNames3 + columnNames1 + columnNames2
meanNames = ["mean"+s for s in columnNames]
stdNames = ["std"+s for s in columnNames]
fieldnames = ["nbNodes", "Mode"] + [val for pair in zip(meanNames, stdNames) for val in pair]
# key: (nbNodes, mode)
# value: a list with all the values
agregate_results = defaultdict(list)
def launch_experiments(ag_results, mode, nbNodes, nbRuns, proba_edge):
folderName = str(nb)+mode
os.makedirs(folderName, exist_ok=True)
os.chdir(folderName)
print("Experiment in mode ", mode, " with ", nbNodes, " nodes ", end='')
if nbRuns > 0:
print("with ", nbRuns, " runs")
for i in trange(nbRuns):
subprocess.run(os.path.join(os.path.dirname(sys.path[0]), programPath) + " " + mode + " " + str(nbNodes) + " " + str(proba_edge) + " 2>> errors.txt", stdout=subprocess.DEVNULL, shell=True)
#subprocess.run(programPath + " " + mode + " " + str(nbNodes), check=True, stdout=subprocess.DEVNULL, shell=True)
else:
print("without runs: reusing results from previous invocation")
# TODO: rather do that as a 3rd phase? So that it's possible to relaunch experiments and have them included
results=[]
files= glob.glob("complex_graph*.csv")
print("Collating results")
for f in tqdm(files):
#data = np.genfromtxt(f, delimiter="\t", encoding=None, dtype=[('Quality', '<i8'), ('Budget', '<i8'), ('ExpectRemainingTime', '<i8'), ('Deadline', '<i8'), ('NbNodes', '<i8'), ('ExecutionTime', '<i8'), ('ChoosingDuration', '<i8'), ('CallbackFlags', 'S16')], names=True)
data = np.genfromtxt(f, delimiter="\t", encoding=None, names=True, skip_header=1, dtype=[('Quality', '<f8'), ('Budget', '<f8'), ('ExpectRemainingTime', '<f8'), ('Deadline', '<f8'), ('NbDegradedNodes', '<f8'), ('NbResamplers', '<f8'),
('ExecutionTime', '<f8'), ('ChoosingDuration', '<f8'), ('CallbackFlags', '<U7')])
#data = np.genfromtxt(f, delimiter="\t", encoding=None, dtype=None, names=True, skip_header=1)
nbActualNodes=-1
nbActualEdges=-1
with open(f, "r") as datafile:
line1 = datafile.readline().split(' ')
nbActualNodes = int(line1[0]) #Always equal to nbNodes byconstruction of the random graph
nbActualEdges = int(line1[1])
nbCycles = data.size #data should be 1D (each element is a dictionary)
degraded = nbCycles -np.count_nonzero(data["Quality"])
ag_results[(nbNodes, mode)].append((data, nbActualEdges, nbCycles, degraded))
os.chdir("..")
def agregate(ag_results):
results = []
print("Stats on results")
bar = tqdm(ag_results.items())
for (nN, mode),info in bar :
res, nbEdges, nbCycles, degraded = zip(*info)
#Concatenate all the ndarrays for this configuration
data = np.concatenate(res)
result={}
for columnName in columnNames1:
bar.write(columnName)
result["mean" + columnName] = data[columnName].mean(dtype=np.float64)
result["std" + columnName] = data[columnName].std(dtype=np.float64,ddof=1)
for columnName in columnNames2:
bar.write(columnName)
degraded_ones = data[columnName][np.nonzero(data[columnName])]
if degraded_ones.size == 0:#If nothing is degraded, we say that the mean is 0
result["mean" + columnName] = 0.
result["std" + columnName] = 0.
else:
result["mean" + columnName] = np.nanmean(degraded_ones, dtype=np.float64)
result["std" + columnName] = np.nanstd(degraded_ones, dtype=np.float64, ddof=1)
print(result["meanChoosingDuration"])
nbCycles = np.array(nbCycles)
result["meanNbCycles"] = nbCycles.mean(dtype=np.float64)
result["stdNbCycles"] = nbCycles.std(dtype=np.float64, ddof=1)
result["nbNodes"] = nN
result["Mode"] = mode
nbEdges= np.array(nbEdges)
result["meanNbEdges"] = nbEdges.mean(dtype=np.float64)
result["stdNbEdges"] = nbEdges.std(dtype=np.float64, ddof=1)
degraded = np.array(degraded)
result["meanNbDegradedCycles"] = degraded.mean(dtype=np.float64)
result["stdNbDegradedCycles"] = degraded.std(dtype=np.float64, ddof=1)
results.append(result)
return results
if len(sys.argv) < 3:
print("Usage: experiments.py destinationFolder nbRuns [proba_edge]")
print("\tif nbRuns <= 0, uses the csv already computed")
sys.exit(1)
# Prepare folder for experiments
# Must be in the base directory of the source
baseFolder = sys.argv[1]
os.makedirs(baseFolder, exist_ok=True)
os.chdir(baseFolder)
# Nb runs per config (mode, nbNodes)
nbRuns = int(sys.argv[2])
proba_edge = 0.5
if len(sys.argv) > 3:
proba_edge = float(sys.argv[3])
#programPath="audio_adaptive_scheduling/target/release/complex_graph"
programPath="audio_adaptive_scheduling/target/release/complex_graph"
#nbNodes = [10, 100, 1000]
#nbNodes = [10, 100, 200, 300, 350, 400, 1000]
#nbNodes = [10, 100, 300, 400, 500, 1000]
#nbNodes = [10, 300]
nbNodes = [10,400,1000]
print("##### Experiments starting in folder ", baseFolder, " with ", nbRuns, " runs per experiment #####\n")
# Launch experiments
for nb in nbNodes:
launch_experiments(agregate_results, "BASE", nb, nbRuns, proba_edge)
launch_experiments(agregate_results, "EX", nb, nbRuns, proba_edge)
launch_experiments(agregate_results, "PROG", nb, nbRuns, proba_edge)
results = agregate(agregate_results)
results.sort(key=lambda res : res["nbNodes"])
print("Writing final result file")
with open(baseFolder+".tsv", 'w') as tsvfile:
writer = csv.DictWriter(tsvfile, fieldnames,dialect="excel-tab")
writer.writeheader()
for result in tqdm(results):
writer.writerow(result)