Skip to content

Commit

Permalink
finalize average emissions dashboard
Browse files Browse the repository at this point in the history
  • Loading branch information
@simei94
simei94 committed May 22, 2024
1 parent 7e21bf3 commit a06e7d8
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 39 deletions.
68 changes: 39 additions & 29 deletions src/main/java/org/matsim/analysis/postAnalysis/EmissionsPostProcessingAverageAnalysis.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;

import static org.matsim.application.ApplicationUtils.globFile;

Expand Down Expand Up @@ -47,7 +48,10 @@ public class EmissionsPostProcessingAverageAnalysis implements MATSimAppCommand
private final Map<Map.Entry<Double, Coord>, Double> meanGridPerHour = new HashMap<>();

private final CsvOptions csv = new CsvOptions();
String value = "value";
private static final String VALUE = "value";
private static final String LINK_ID = "linkId";
private static final String POLLUTANT = "Pollutant";
private static final String ANALYSIS_DIR = "/analysis/emissions";

public static void main(String[] args) {
new EmissionsPostProcessingAverageAnalysis().execute(args);
Expand All @@ -58,46 +62,53 @@ public Integer call() throws Exception {

String runs = input.getPath("runs");

// function to determine column types
Function<String, ColumnType> columnTypeFunction = columnName -> {
if (columnName.equals(LINK_ID) || columnName.equals(POLLUTANT)) {
return ColumnType.STRING;
} else {
return ColumnType.DOUBLE;
}
};

List<String> foldersSeeded = Arrays.stream(runs.split(",")).toList();

// add stats from every run to map
for (String folder : foldersSeeded) {
String totalCsv = globFile(Path.of(folder + "/analysis/emissions" ), "*emissions_total.csv*").toString();
String emissionsPerLinkMCsv = globFile(Path.of(folder + "/analysis/emissions"), "*emissions_per_link_per_m.csv*").toString();
String emissionsGridPerDayCsv = globFile(Path.of(folder + "/analysis/emissions"), "*emissions_grid_per_day.xyt.csv*").toString();
String emissionsGridPerHourCsv = globFile(Path.of(folder + "/analysis/emissions"), "*emissions_grid_per_hour.csv*").toString();
final Path analysisDir = Path.of(folder + ANALYSIS_DIR);
String totalCsv = globFile(analysisDir, "*emissions_total.csv*").toString();
String emissionsPerLinkMCsv = globFile(analysisDir, "*emissions_per_link_per_m.csv*").toString();
String emissionsGridPerDayCsv = globFile(analysisDir, "*emissions_grid_per_day.xyt.csv*").toString();
String emissionsGridPerHourCsv = globFile(analysisDir, "*emissions_grid_per_hour.csv*").toString();

Table total = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(totalCsv))
.columnTypes(columnTypeFunction)
.sample(false)
.separator(csv.detectDelimiter(totalCsv)).build());

Table emissionsPerLinkM = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsPerLinkMCsv))
.columnTypes(columnTypeFunction)
.sample(false)
.separator(csv.detectDelimiter(emissionsPerLinkMCsv)).build());

// TODO: update matsim version to newest for necessary changes in detectDelimiter method
Table emissionsGridPerDay = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsGridPerDayCsv))
.columnTypes(columnTypeFunction)
.sample(false)
.separator(csv.detectDelimiter(emissionsGridPerDayCsv)).header(true).build());

Table emissionsGridPerHour = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsGridPerHourCsv))
.columnTypes(columnTypeFunction)
.sample(false)
.separator(csv.detectDelimiter(emissionsGridPerHourCsv)).build());

// get all total stats
for (int i = 0; i < total.rowCount(); i++) {
Row row = total.row(i);

if (!totalStats.containsKey(row.getString(i))) {
totalStats.put(row.getString(i), new ArrayList<>());
}

// some values are in format hh:mm:ss or empty
if (row.getString("kg").isEmpty()) {
totalStats.get(row.getString("Pollutant")).add(0.);
} else {
totalStats.get(row.getString("Pollutant")).add(row.getDouble("kg"));
if (!totalStats.containsKey(row.getString(POLLUTANT))) {
totalStats.put(row.getString(POLLUTANT), new ArrayList<>());
}
totalStats.get(row.getString(POLLUTANT)).add(row.getDouble("kg"));
}

// get all per link per m stats
Expand All @@ -106,18 +117,13 @@ public Integer call() throws Exception {
Double[] values = new Double[emissionsPerLinkM.columnCount() - 1];

// iterate through columns. this file contains 23 params per link, as of may24
for (int j = 1; i < emissionsPerLinkM.columnCount() - 1; j++) {
if (!perLinkMStats.containsKey(row.getString(i))) {
perLinkMStats.put(row.getString(i), new ArrayList<>());
}

if (row.getColumnType(j) == ColumnType.INTEGER) {
values[j - 1] = (double) row.getInt(j);
} else {
values[j - 1] = row.getDouble(j);
for (int j = 1; j < emissionsPerLinkM.columnCount(); j++) {
if (!perLinkMStats.containsKey(row.getString(LINK_ID))) {
perLinkMStats.put(row.getString(LINK_ID), new ArrayList<>());
}
values[j - 1] = row.getDouble(j);
}
perLinkMStats.get(row.getString(i)).add(values);
perLinkMStats.get(row.getString(LINK_ID)).add(values);
}

// get all grid per day stats
Expand All @@ -142,6 +148,10 @@ public Integer call() throws Exception {

for (Double[] d : e.getValue()) {
for (int i = 0; i <= d.length - 1; i++) {
// initial array values are null
if (sums[i] == null) {
sums[i] = 0.;
}
sums[i] += d[i];
}
}
Expand All @@ -160,7 +170,7 @@ public Integer call() throws Exception {

// write total mean stats
try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_emissions_total.csv")), CSVFormat.DEFAULT)) {
printer.printRecord("Pollutant", "kg");
printer.printRecord(POLLUTANT, "kg");

for (Map.Entry<String, Double> e : meanTotal.entrySet()) {
printer.printRecord("mean-" + e.getKey(), e.getValue());
Expand All @@ -169,7 +179,7 @@ public Integer call() throws Exception {

// write per linkM mean stats
try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_emissions_per_link_per_m.csv")), CSVFormat.DEFAULT)) {
printer.printRecord("linkId", "CO [g/m]", "CO2_TOTAL [g/m]", "FC [g/m]", "HC [g/m]", "NMHC [g/m]", "NOx [g/m]", "NO2 [g/m]", "PM [g/m]", "SO2 [g/m]",
printer.printRecord(LINK_ID, "CO [g/m]", "CO2_TOTAL [g/m]", "FC [g/m]", "HC [g/m]", "NMHC [g/m]", "NOx [g/m]", "NO2 [g/m]", "PM [g/m]", "SO2 [g/m]",
"FC_MJ [g/m]", "CO2_rep [g/m]", "CO2e [g/m]", "PM2_5 [g/m]", "PM2_5_non_exhaust [g/m]", "PM_non_exhaust [g/m]", "BC_exhaust [g/m]", "BC_non_exhaust [g/m]",
"Benzene [g/m]", "PN [g/m]", "Pb [g/m]", "CH4 [g/m]", "N2O [g/m]", "NH3 [g/m]"
);
Expand Down Expand Up @@ -204,15 +214,15 @@ private void getGridData(Table gridTable, Map<Map.Entry<Double, Coord>, List<Dou
Map.Entry<Double, Coord> entry = new AbstractMap.SimpleEntry<>(row.getDouble("time"), new Coord(row.getDouble("x"), row.getDouble("y")));

dataMap.computeIfAbsent(entry, key -> new ArrayList<>());
dataMap.get(entry).add(row.getDouble(value));
dataMap.get(entry).add(row.getDouble(VALUE));
}
}

private void writeGridFile(String fileName, Map<Map.Entry<Double, Coord>, Double> values) throws IOException {
try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath(fileName)), CSVFormat.DEFAULT)) {

printer.printRecord("# EPSG:25832");
printer.printRecord("time", "x", "y", value);
printer.printRecord("time", "x", "y", VALUE);

for (Map.Entry<Map.Entry<Double, Coord>, Double> e : values.entrySet()) {
printer.printRecord(e.getKey().getKey(), e.getKey().getValue().getX(), e.getKey().getValue().getY(), e.getValue());
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/org/matsim/dashboard/AverageDrtDashboard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ private String postProcess(Data data, String outputFile) {
@Override
public void configure(Header header, Layout layout) {
header.title = mode;
header.description = "Overview for the demand-responsive mode '" + mode + "'" +
"/n" + "This dashboard shows average values for " + noRuns +
header.description = "Overview for the demand-responsive mode '" + mode + "'. This dashboard shows average values for " + noRuns +
" simulation runs. For the results of the specific runs please choose the according directory next to this dashboard.yaml.";

// DEMAND
Expand Down
33 changes: 27 additions & 6 deletions src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
package org.matsim.dashboard;

import org.matsim.analysis.postAnalysis.EmissionsPostProcessingAverageAnalysis;
import org.matsim.analysis.postAnalysis.drt.DrtPostProcessingAverageAnalysis;
import org.matsim.application.prepare.network.CreateGeoJsonNetwork;
import org.matsim.simwrapper.Dashboard;
import org.matsim.simwrapper.Data;
import org.matsim.simwrapper.Header;
Expand All @@ -31,13 +29,18 @@
import org.matsim.simwrapper.viz.Links;
import org.matsim.simwrapper.viz.Table;

import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

/**
* Average emissions dashboard for several runs with the same config but a different random seed.
*/
public class AverageKelheimEmissionsDashboard implements Dashboard{
public class AverageKelheimEmissionsDashboard implements Dashboard {
private final List<String> dirs;
private final Integer noRuns;
private final String pathToCsvBase;
Expand Down Expand Up @@ -65,6 +68,24 @@ private String postProcess(Data data, String outputFile) {
return data.compute(EmissionsPostProcessingAverageAnalysis.class, outputFile, args.toArray(new String[0]));
}

private String copyGeoJsonNetwork() {

for (String dir : dirs) {
File networkFile = new File(dir + "/analysis/network/network.geojson");
Path target = Path.of(Path.of(dir).getParent() + "/analysis/network");

if (Files.notExists(target) && networkFile.exists() && networkFile.isFile()) {
try {
Files.createDirectories(target);
Files.copy(networkFile.toPath(), Path.of(target + "/network.geojson"));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
}
return "analysis/network/network.geojson";
}

/**
* Produces the dashboard.
*/
Expand All @@ -74,7 +95,7 @@ public void configure(Header header, Layout layout) {

String linkDescription = "Displays the emissions for each link per meter. Be aware that emission values are provided in the simulation sample size!";
if (pathToCsvBase != null){
linkDescription += String.format("\n Base is %s", pathToCsvBase);
linkDescription += String.format("%n Base is %s", pathToCsvBase);
}
String finalLinkDescription = linkDescription;

Expand All @@ -92,8 +113,8 @@ public void configure(Header header, Layout layout) {
viz.description = finalLinkDescription;
viz.height = 12.0;
viz.datasets.csvFile = postProcess(data, "mean_emissions_per_link_per_m.csv");
viz.datasets.csvBase = this.pathToCsvBase;
viz.network = data.compute(CreateGeoJsonNetwork.class, "network.geojson");
viz.datasets.csvBase = Path.of(this.dirs.get(0)).getParent().relativize(Path.of(pathToCsvBase)).toString();
viz.network = copyGeoJsonNetwork();
viz.display.color.columnName = "CO2_TOTAL [g/m]";
viz.display.color.dataset = "csvFile";
viz.display.width.scaleFactor = 100;
Expand Down
2 changes: 0 additions & 2 deletions src/main/java/org/matsim/dashboard/CreateAverageDashboards.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,6 @@ public Integer call() throws Exception {
}

sw.addDashboard(Dashboard.customize(new AverageKelheimEmissionsDashboard(foldersSeeded, noRuns, pathToBaseRun)).context("emissions"));

// TODO: rather call generate method with append true than the standard one bc we are in post processing
sw.generate(Path.of(inputPath), true);
sw.run(Path.of(inputPath));

Expand Down

0 comments on commit a06e7d8

Please sign in to comment.