Skip to content

Commit

Permalink
Clean up the logging output
Browse files Browse the repository at this point in the history
  • Loading branch information
@mfitz
mfitz committed Mar 5, 2024
1 parent f34de6d commit 0fb0fc6
Showing 1 changed file with 29 additions and 24 deletions.
53 changes: 29 additions & 24 deletions src/main/java/com/arup/cml/abm/kpi/tablesaw/TablesawKpiCalculator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -628,7 +628,7 @@ public Table addPTAccessColumnWithinDistance(Table table, Table stops, double di
double x = stopRow.getNumber("x");
double y = stopRow.getNumber("y");

LOGGER.info("Calculating access for {} persons for stop {}",
LOGGER.debug("Calculating access for {} persons for stop {}",
table.rowCount(), stopRow.getString("name"));
table.addColumns(
table.doubleColumn("x").subtract(x).power(2)
Expand Down Expand Up @@ -713,34 +713,34 @@ public Table writeCongestionKpi(Path outputDirectory) {
public double writeMobilitySpaceUsageKpi(Path outputDirectory) {
LOGGER.info("Writing Mobility Space Usage KPI to {}", outputDirectory);

LOGGER.info("Filtering the activities table, which contains {} rows, for car activities",
LOGGER.debug("Filtering the activities table, which contains {} rows, for car activities",
activities.rowCount());
Table carActivities = activities
.where(activities.stringColumn("access_mode").isEqualTo("car")
.or(activities.stringColumn("egress_mode").isEqualTo("car")));
LOGGER.info("Filtered down to {} car activity rows", carActivities.rowCount());
LOGGER.debug("Filtered activities table down to {} car activity rows", carActivities.rowCount());

LOGGER.info("Make a table for peopleInFacilities");
LOGGER.debug("Making a table for peopleInFacilities");
Table peopleInFacilities = carActivities
.summarize("person", countUnique)
.by("facility_id", "activity_type");
LOGGER.info("Make a table for tripsToFacilities");
LOGGER.debug("Make a table for tripsToFacilities");
Table tripsToFacilities = carActivities
.summarize("access_trip_id", countNonMissing)
.by("facility_id", "activity_type");

LOGGER.info("Joining peopleInFacilities with tripsToFacilities");
LOGGER.debug("Joining peopleInFacilities with tripsToFacilities");
Table intermediate = peopleInFacilities
.joinOn("facility_id", "activity_type")
.inner(tripsToFacilities)
.setName("Mobility Space Usage");
LOGGER.info("Join complete");
LOGGER.debug("Join complete");
intermediate.column("Count Unique [person]").setName("max_occupancy");
intermediate.column("Count [access_trip_id]").setName("total_trips");

// https://www.interparking-france.com/en/what-are-the-dimensions-of-a-parking-space/
LOGGER.info("Calculating parking space demand with parking factor: 11.5");
LOGGER.info("Adding a new parking_space_demand column to the joined table, derived from the max_occupancy");
LOGGER.debug("Adding a new parking_space_demand column to the joined table, derived from the max_occupancy");
intermediate.addColumns(
intermediate.numberColumn("max_occupancy")
.multiply(11.5)
Expand All @@ -758,14 +758,14 @@ public double writeMobilitySpaceUsageKpi(Path outputDirectory) {
kpi.column("Sum [parking_space_demand]").setName("parking_space_demand");
kpi.column("Sum [total_trips]").setName("total_trips");

LOGGER.info("Adding new weighted_demand column to the KPI table");
LOGGER.debug("Adding new weighted_demand column to the KPI table");
kpi.addColumns(
kpi.numberColumn("parking_space_demand")
.multiply(kpi.numberColumn("total_trips")
.divide(kpi.numberColumn("total_trips").sum()))
.setName("weighted_demand")
);
LOGGER.info("Finished adding weighted_demand column to the KPI table");
LOGGER.debug("Finished adding weighted_demand column to the KPI table");
this.writeTableCompressed(intermediate,
String.format("%s/kpi-mobility-space-usage-per-activity-type.csv", outputDirectory),
this.compressionType);
Expand Down Expand Up @@ -804,9 +804,9 @@ private double round(double number, int decimalPoints) {
private Table sanitiseInfiniteColumnValuesInTable(Table table, DoubleColumn column) {
Table infiniteValuesTable = table.where(column.eval(Double::isInfinite));
if (!infiniteValuesTable.isEmpty()) {
LOGGER.warn(("Table: `%s` has %d row(s) affected by infinite values in column: `%s`. " +
"These rows will be dropped for this calculation.")
.formatted(table.name(), infiniteValuesTable.rowCount(), column.name()));
LOGGER.warn("Table: '{}' has {} row(s) affected by infinite values in column: '{}'. " +
"These rows will be dropped for this calculation.",
table.name(), infiniteValuesTable.rowCount(), column.name());
return table.dropWhere(column.eval(Double::isInfinite));
} else {
return table;
Expand Down Expand Up @@ -924,7 +924,7 @@ private Table addCostToLegs(Table legs, Table personModeScores, MoneyLog moneyLo
);
personModeScores.column("score_mode").setName("mode");

LOGGER.info("Compute monetary cost for each leg from scoring params");
LOGGER.debug("Computing monetary cost for each leg from scoring params");
legs.addColumns(
legs.intColumn("distance")
.multiply(legs.doubleColumn("monetaryDistanceRate"))
Expand All @@ -934,14 +934,14 @@ private Table addCostToLegs(Table legs, Table personModeScores, MoneyLog moneyLo
legs.removeColumns("monetaryDistanceRate", "dailyMonetaryConstant");

LOGGER.info("Adding contribution from person money events");
LOGGER.info("Create a time columns in seconds");
LOGGER.debug("Create a time columns in seconds");
DoubleColumn dep_time_seconds = DoubleColumn.create("dep_time_seconds");
DoubleColumn trav_time_seconds = DoubleColumn.create("trav_time_seconds");
LOGGER.info("Adding dep_time column");
LOGGER.debug("Adding dep_time column");
legs.stringColumn("dep_time")
.forEach(time -> dep_time_seconds.append(
(int) Time.parseTime(time)));
LOGGER.info("Adding trav_time column");
LOGGER.debug("Adding trav_time column");
legs.stringColumn("trav_time")
.forEach(time -> trav_time_seconds.append(
(int) Time.parseTime(time)));
Expand All @@ -960,7 +960,7 @@ private Table addCostToLegs(Table legs, Table personModeScores, MoneyLog moneyLo
);
}
}
LOGGER.info("Finished iterating over the money log");
LOGGER.debug("Finished iterating over the money log");
legs.removeColumns(dep_time_seconds, arr_time_seconds);
LOGGER.info("Finished adding costs to legs table");
return legs;
Expand Down Expand Up @@ -1006,6 +1006,10 @@ private Table createActivitiesTable(Table trips) {
uniquePersons.countUnique(),
trips.rowCount());
int personsProcessedCount = 0;
int loggingStepSize = 10000;
if (uniquePersons.countUnique() > 10) {
loggingStepSize = uniquePersons.countUnique() / 10;
}
for (String person : uniquePersons) {
Table personTrips = trips
.where(trips.stringColumn("person").isEqualTo(person))
Expand Down Expand Up @@ -1050,7 +1054,7 @@ private Table createActivitiesTable(Table trips) {
// finally append the activities of this person
activities.append(personActivities);
personsProcessedCount++;
if (personsProcessedCount % 10000 == 0) {
if (personsProcessedCount % loggingStepSize == 0) {
LOGGER.info("Created activities for {} persons so far", personsProcessedCount);
}
}
Expand Down Expand Up @@ -1093,7 +1097,7 @@ private void createPeopleTables(InputStream personInputStream, ScoringConfigGrou
DoubleColumn monetaryDistanceRateColumn = DoubleColumn.create("monetaryDistanceRate");
DoubleColumn dailyMonetaryConstantColumn = DoubleColumn.create("dailyMonetaryConstant");

LOGGER.info("Adding people to the person mode scores table");
LOGGER.debug("Adding people to the person mode scores table");
for (Row row : personModeScores) {
String person = row.getString("person");
String subpopulation = row.getString("subpopulation");
Expand All @@ -1105,7 +1109,7 @@ private void createPeopleTables(InputStream personInputStream, ScoringConfigGrou
dailyMonetaryConstantColumn.append(modeParams.getDailyMonetaryConstant());
}
}
LOGGER.info("Joining person mode scores table to a new temp table");
LOGGER.debug("Joining person mode scores table to a new temp table");
personModeScores = personModeScores
.joinOn("person")
.rightOuter(Table
Expand Down Expand Up @@ -1240,6 +1244,7 @@ private void createTransitTables(TransitSchedule schedule) {
routeIDColumn,
modeColumn
);
LOGGER.info("Finished creating Transit Tables");
}

private void createVehicleTable(Vehicles inputVehicles) {
Expand Down Expand Up @@ -1406,13 +1411,13 @@ public Table readCSVInputStream(InputStream inputStream, Map<String, ColumnType>
}

private void writeContentToFile(String path, String content, CompressionType compressionType) {
LOGGER.info(String.format("Writing file %s", path));
LOGGER.info("Writing file {}", path);
try (Writer wr = IOUtils.getBufferedWriter(path.concat(compressionType.fileEnding))) {
wr.write(content);
} catch (IOException e) {
LOGGER.error("!!! Failed to save content '{}' to file: '{}'", content, path);
}
LOGGER.info(String.format("Finished writing file %s", path));
LOGGER.info("Finished writing file {}", path);
}

private void writeSupportingData(Path outputDir) {
Expand All @@ -1436,7 +1441,7 @@ private void writeSupportingData(Path outputDir) {
this.writeTableCompressed(scheduleStops, String.format("%s/supporting-data-scheduleStops.csv", outputDir), this.compressionType);
this.writeTableCompressed(scheduleRoutes, String.format("%s/supporting-data-scheduleRoutes.csv", outputDir), this.compressionType);
this.writeTableCompressed(vehicles, String.format("%s/supporting-data-vehicles.csv", outputDir), this.compressionType);
LOGGER.info("Finished writing supporting data files");
LOGGER.info("Finished writing supporting data files");
}

private OutputStream getCompressedOutputStream(String filepath, CompressionType compressionType) {
Expand Down

0 comments on commit 0fb0fc6

Please sign in to comment.