From a06e7d8a73ba7e3f386e6eadc245dbf17c042964 Mon Sep 17 00:00:00 2001 From: simei94 Date: Wed, 22 May 2024 14:31:41 +0200 Subject: [PATCH] finalize average emissions dashboard --- ...missionsPostProcessingAverageAnalysis.java | 68 +++++++++++-------- .../matsim/dashboard/AverageDrtDashboard.java | 3 +- .../AverageKelheimEmissionsDashboard.java | 33 +++++++-- .../dashboard/CreateAverageDashboards.java | 2 - 4 files changed, 67 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/matsim/analysis/postAnalysis/EmissionsPostProcessingAverageAnalysis.java b/src/main/java/org/matsim/analysis/postAnalysis/EmissionsPostProcessingAverageAnalysis.java index fbc9895c..ce66b344 100644 --- a/src/main/java/org/matsim/analysis/postAnalysis/EmissionsPostProcessingAverageAnalysis.java +++ b/src/main/java/org/matsim/analysis/postAnalysis/EmissionsPostProcessingAverageAnalysis.java @@ -20,6 +20,7 @@ import java.nio.file.Path; import java.util.*; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import static org.matsim.application.ApplicationUtils.globFile; @@ -47,7 +48,10 @@ public class EmissionsPostProcessingAverageAnalysis implements MATSimAppCommand private final Map, Double> meanGridPerHour = new HashMap<>(); private final CsvOptions csv = new CsvOptions(); - String value = "value"; + private static final String VALUE = "value"; + private static final String LINK_ID = "linkId"; + private static final String POLLUTANT = "Pollutant"; + private static final String ANALYSIS_DIR = "/analysis/emissions"; public static void main(String[] args) { new EmissionsPostProcessingAverageAnalysis().execute(args); @@ -58,29 +62,42 @@ public Integer call() throws Exception { String runs = input.getPath("runs"); +// function to determine column types + Function columnTypeFunction = columnName -> { + if (columnName.equals(LINK_ID) || columnName.equals(POLLUTANT)) { + return ColumnType.STRING; + } else { + return ColumnType.DOUBLE; + } + }; + List foldersSeeded = Arrays.stream(runs.split(",")).toList(); // add stats from every run to map for (String folder : foldersSeeded) { - String totalCsv = globFile(Path.of(folder + "/analysis/emissions" ), "*emissions_total.csv*").toString(); - String emissionsPerLinkMCsv = globFile(Path.of(folder + "/analysis/emissions"), "*emissions_per_link_per_m.csv*").toString(); - String emissionsGridPerDayCsv = globFile(Path.of(folder + "/analysis/emissions"), "*emissions_grid_per_day.xyt.csv*").toString(); - String emissionsGridPerHourCsv = globFile(Path.of(folder + "/analysis/emissions"), "*emissions_grid_per_hour.csv*").toString(); + final Path analysisDir = Path.of(folder + ANALYSIS_DIR); + String totalCsv = globFile(analysisDir, "*emissions_total.csv*").toString(); + String emissionsPerLinkMCsv = globFile(analysisDir, "*emissions_per_link_per_m.csv*").toString(); + String emissionsGridPerDayCsv = globFile(analysisDir, "*emissions_grid_per_day.xyt.csv*").toString(); + String emissionsGridPerHourCsv = globFile(analysisDir, "*emissions_grid_per_hour.csv*").toString(); Table total = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(totalCsv)) + .columnTypes(columnTypeFunction) .sample(false) .separator(csv.detectDelimiter(totalCsv)).build()); Table emissionsPerLinkM = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsPerLinkMCsv)) + .columnTypes(columnTypeFunction) .sample(false) .separator(csv.detectDelimiter(emissionsPerLinkMCsv)).build()); -// TODO: update matsim version to newest for necessary changes in detectDelimiter method Table emissionsGridPerDay = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsGridPerDayCsv)) + .columnTypes(columnTypeFunction) .sample(false) .separator(csv.detectDelimiter(emissionsGridPerDayCsv)).header(true).build()); Table emissionsGridPerHour = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(emissionsGridPerHourCsv)) + .columnTypes(columnTypeFunction) .sample(false) .separator(csv.detectDelimiter(emissionsGridPerHourCsv)).build()); @@ -88,16 +105,10 @@ public Integer call() throws Exception { for (int i = 0; i < total.rowCount(); i++) { Row row = total.row(i); - if (!totalStats.containsKey(row.getString(i))) { - totalStats.put(row.getString(i), new ArrayList<>()); - } - -// some values are in format hh:mm:ss or empty - if (row.getString("kg").isEmpty()) { - totalStats.get(row.getString("Pollutant")).add(0.); - } else { - totalStats.get(row.getString("Pollutant")).add(row.getDouble("kg")); + if (!totalStats.containsKey(row.getString(POLLUTANT))) { + totalStats.put(row.getString(POLLUTANT), new ArrayList<>()); } + totalStats.get(row.getString(POLLUTANT)).add(row.getDouble("kg")); } // get all per link per m stats @@ -106,18 +117,13 @@ public Integer call() throws Exception { Double[] values = new Double[emissionsPerLinkM.columnCount() - 1]; // iterate through columns. this file contains 23 params per link, as of may24 - for (int j = 1; i < emissionsPerLinkM.columnCount() - 1; j++) { - if (!perLinkMStats.containsKey(row.getString(i))) { - perLinkMStats.put(row.getString(i), new ArrayList<>()); - } - - if (row.getColumnType(j) == ColumnType.INTEGER) { - values[j - 1] = (double) row.getInt(j); - } else { - values[j - 1] = row.getDouble(j); + for (int j = 1; j < emissionsPerLinkM.columnCount(); j++) { + if (!perLinkMStats.containsKey(row.getString(LINK_ID))) { + perLinkMStats.put(row.getString(LINK_ID), new ArrayList<>()); } + values[j - 1] = row.getDouble(j); } - perLinkMStats.get(row.getString(i)).add(values); + perLinkMStats.get(row.getString(LINK_ID)).add(values); } // get all grid per day stats @@ -142,6 +148,10 @@ public Integer call() throws Exception { for (Double[] d : e.getValue()) { for (int i = 0; i <= d.length - 1; i++) { +// initial array values are null + if (sums[i] == null) { + sums[i] = 0.; + } sums[i] += d[i]; } } @@ -160,7 +170,7 @@ public Integer call() throws Exception { // write total mean stats try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_emissions_total.csv")), CSVFormat.DEFAULT)) { - printer.printRecord("Pollutant", "kg"); + printer.printRecord(POLLUTANT, "kg"); for (Map.Entry e : meanTotal.entrySet()) { printer.printRecord("mean-" + e.getKey(), e.getValue()); @@ -169,7 +179,7 @@ public Integer call() throws Exception { // write per linkM mean stats try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("mean_emissions_per_link_per_m.csv")), CSVFormat.DEFAULT)) { - printer.printRecord("linkId", "CO [g/m]", "CO2_TOTAL [g/m]", "FC [g/m]", "HC [g/m]", "NMHC [g/m]", "NOx [g/m]", "NO2 [g/m]", "PM [g/m]", "SO2 [g/m]", + printer.printRecord(LINK_ID, "CO [g/m]", "CO2_TOTAL [g/m]", "FC [g/m]", "HC [g/m]", "NMHC [g/m]", "NOx [g/m]", "NO2 [g/m]", "PM [g/m]", "SO2 [g/m]", "FC_MJ [g/m]", "CO2_rep [g/m]", "CO2e [g/m]", "PM2_5 [g/m]", "PM2_5_non_exhaust [g/m]", "PM_non_exhaust [g/m]", "BC_exhaust [g/m]", "BC_non_exhaust [g/m]", "Benzene [g/m]", "PN [g/m]", "Pb [g/m]", "CH4 [g/m]", "N2O [g/m]", "NH3 [g/m]" ); @@ -204,7 +214,7 @@ private void getGridData(Table gridTable, Map, List entry = new AbstractMap.SimpleEntry<>(row.getDouble("time"), new Coord(row.getDouble("x"), row.getDouble("y"))); dataMap.computeIfAbsent(entry, key -> new ArrayList<>()); - dataMap.get(entry).add(row.getDouble(value)); + dataMap.get(entry).add(row.getDouble(VALUE)); } } @@ -212,7 +222,7 @@ private void writeGridFile(String fileName, Map, Double try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath(fileName)), CSVFormat.DEFAULT)) { printer.printRecord("# EPSG:25832"); - printer.printRecord("time", "x", "y", value); + printer.printRecord("time", "x", "y", VALUE); for (Map.Entry, Double> e : values.entrySet()) { printer.printRecord(e.getKey().getKey(), e.getKey().getValue().getX(), e.getKey().getValue().getY(), e.getValue()); diff --git a/src/main/java/org/matsim/dashboard/AverageDrtDashboard.java b/src/main/java/org/matsim/dashboard/AverageDrtDashboard.java index 6edf3461..df880d8e 100644 --- a/src/main/java/org/matsim/dashboard/AverageDrtDashboard.java +++ b/src/main/java/org/matsim/dashboard/AverageDrtDashboard.java @@ -36,8 +36,7 @@ private String postProcess(Data data, String outputFile) { @Override public void configure(Header header, Layout layout) { header.title = mode; - header.description = "Overview for the demand-responsive mode '" + mode + "'" + - "/n" + "This dashboard shows average values for " + noRuns + + header.description = "Overview for the demand-responsive mode '" + mode + "'. This dashboard shows average values for " + noRuns + " simulation runs. For the results of the specific runs please choose the according directory next to this dashboard.yaml."; // DEMAND diff --git a/src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java b/src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java index e5dc6e81..40c691fe 100644 --- a/src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java +++ b/src/main/java/org/matsim/dashboard/AverageKelheimEmissionsDashboard.java @@ -21,8 +21,6 @@ package org.matsim.dashboard; import org.matsim.analysis.postAnalysis.EmissionsPostProcessingAverageAnalysis; -import org.matsim.analysis.postAnalysis.drt.DrtPostProcessingAverageAnalysis; -import org.matsim.application.prepare.network.CreateGeoJsonNetwork; import org.matsim.simwrapper.Dashboard; import org.matsim.simwrapper.Data; import org.matsim.simwrapper.Header; @@ -31,13 +29,18 @@ import org.matsim.simwrapper.viz.Links; import org.matsim.simwrapper.viz.Table; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; /** * Average emissions dashboard for several runs with the same config but a different random seed. */ -public class AverageKelheimEmissionsDashboard implements Dashboard{ +public class AverageKelheimEmissionsDashboard implements Dashboard { private final List dirs; private final Integer noRuns; private final String pathToCsvBase; @@ -65,6 +68,24 @@ private String postProcess(Data data, String outputFile) { return data.compute(EmissionsPostProcessingAverageAnalysis.class, outputFile, args.toArray(new String[0])); } + private String copyGeoJsonNetwork() { + + for (String dir : dirs) { + File networkFile = new File(dir + "/analysis/network/network.geojson"); + Path target = Path.of(Path.of(dir).getParent() + "/analysis/network"); + + if (Files.notExists(target) && networkFile.exists() && networkFile.isFile()) { + try { + Files.createDirectories(target); + Files.copy(networkFile.toPath(), Path.of(target + "/network.geojson")); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + return "analysis/network/network.geojson"; + } + /** * Produces the dashboard. */ @@ -74,7 +95,7 @@ public void configure(Header header, Layout layout) { String linkDescription = "Displays the emissions for each link per meter. Be aware that emission values are provided in the simulation sample size!"; if (pathToCsvBase != null){ - linkDescription += String.format("\n Base is %s", pathToCsvBase); + linkDescription += String.format("%n Base is %s", pathToCsvBase); } String finalLinkDescription = linkDescription; @@ -92,8 +113,8 @@ public void configure(Header header, Layout layout) { viz.description = finalLinkDescription; viz.height = 12.0; viz.datasets.csvFile = postProcess(data, "mean_emissions_per_link_per_m.csv"); - viz.datasets.csvBase = this.pathToCsvBase; - viz.network = data.compute(CreateGeoJsonNetwork.class, "network.geojson"); + viz.datasets.csvBase = Path.of(this.dirs.get(0)).getParent().relativize(Path.of(pathToCsvBase)).toString(); + viz.network = copyGeoJsonNetwork(); viz.display.color.columnName = "CO2_TOTAL [g/m]"; viz.display.color.dataset = "csvFile"; viz.display.width.scaleFactor = 100; diff --git a/src/main/java/org/matsim/dashboard/CreateAverageDashboards.java b/src/main/java/org/matsim/dashboard/CreateAverageDashboards.java index 8c0dfe7a..4b077ed7 100644 --- a/src/main/java/org/matsim/dashboard/CreateAverageDashboards.java +++ b/src/main/java/org/matsim/dashboard/CreateAverageDashboards.java @@ -65,8 +65,6 @@ public Integer call() throws Exception { } sw.addDashboard(Dashboard.customize(new AverageKelheimEmissionsDashboard(foldersSeeded, noRuns, pathToBaseRun)).context("emissions")); - -// TODO: rather call generate method with append true than the standard one bc we are in post processing sw.generate(Path.of(inputPath), true); sw.run(Path.of(inputPath));