Skip to content

Commit

Permalink
feat: get damage details from calfire
Browse files Browse the repository at this point in the history
  • Loading branch information
@advayDev1
advayDev1 committed Oct 27, 2019
1 parent ad85cb8 commit b0d8deb
Show file tree
Hide file tree
Showing 6 changed files with 221 additions and 36 deletions.
16 changes: 16 additions & 0 deletions commands/run.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,11 @@ exports.builder = {
default: true,
desc: 'Whether to require CALFIRE data',
},
parseCalfireDetails: {
boolean: true,
default: true,
desc: 'Whether to parse CALFIRE details',
}
};

exports.handler = (argv) => {
Expand Down Expand Up @@ -726,6 +731,17 @@ exports.handler = (argv) => {
}
}

if (argv.parseCalfireDetails) {
if (cur.Link && cur.Link.startsWith('https://www.fire.ca.gov/')) {
const fd = await calfire.getFireDetail(cur, argv.userAgent);
if (fd) {
cur.Damage = fd;
} else {
delete cur['Damage'];
}
}
}

// This will go through. Make sure we don't put an old update on Twitter in the mean time.
const delPaths = await del([argv.outputdir + '/postqueue/*-' + cur.UniqueFireIdentifier + '-*.yaml']);
if (delPaths.length > 0) {
Expand Down
50 changes: 47 additions & 3 deletions lib/calfire.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ const promisify = require('util').promisify;
const rp = require('request-promise');
const {DateTime} = require('luxon');

const cheerio = require('cheerio');

const parse = promisify(require('csv-parse'));

const dateString = require('./util').dateString;
Expand All @@ -33,6 +35,7 @@ const nfsaMapping = {
incident_name: 'Fire_Name',
incident_date_last_update: 'ModifiedOnDateTime',
incident_date_last_updateEpoch: 'ModifiedOnDateTimeEpoch',
incident_url: 'Link',

// todo:
firecause: 'FireCause',
Expand Down Expand Up @@ -77,9 +80,6 @@ const processFire = function(e, provenance) {
ret2.State = 'CA';
ret2.UniqueFireIdentifier = 'CALFIRE-' + ret2.incident_id;
ret2.Hashtag = util.fireHashTag(ret2.Name);
const link = new URL('https://www.fire.ca.gov/incident/');
link.searchParams.append('incident', ret2.incident_id);
ret2.Link = link.toString();
ret2._Provenance = _.cloneDeep(provenance);
ret2._CorrelationIds = [ret2.UniqueFireIdentifier];
return ret2;
Expand Down Expand Up @@ -133,3 +133,47 @@ exports.getFires = async function(ua) {
return x;
};


exports.getFireDetail = async function(fire, ua) {
const firesDataOptions = {
uri: fire.Link,
qs: qs,
headers: {
'User-Agent': 'Request-Promise; ' + ua,
},
json: false,
};
const provenance = util.createProvenance(firesDataOptions);
const resp = await rp(firesDataOptions);
const parser = cheerio.load(resp);
const dmgElems = parser('h4', '.incident-damages-and-losses');
const dmgTexts = _.map(dmgElems, (x) => (parser(x).text()));
const ret = {
};
let some = false;
_.forEach(dmgTexts, (x) => {
const m = x.match(/((?<dmg>\d+) Structures? Damaged)|((?<dest>\d+) Structures? Destroyed)|((?<inj>\d+) Injur.*)|((?<fatal>\d+) Fatal.*)/);
if (m && m.groups) {
if (m.groups.dmg) {
ret.structuresDamaged = Number.parseInt(m.groups.dmg);
some = true;
}
if (m.groups.dest) {
ret.structuresDestroyed = Number.parseInt(m.groups.dest);
some = true;
}
if (m.groups.inj) {
ret.injuries = Number.parseInt(m.groups.inj);
some = true;
}
if (m.groups.fatal) {
ret.fatalities = Number.parseInt(m.groups.fatal);
some = true;
}
}
});
if (!some) {
return null;
}
return ret;
};
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
"@turf/circle": "^6.0.1",
"@turf/distance": "^6.0.1",
"all-the-cities": "https://github.com/publicdocs-platform/us-cities-db",
"cheerio": "^1.0.0-rc.3",
"csv": "^5.1.3",
"deep-diff": "^1.0.2",
"del": "^5.1.0",
Expand Down
56 changes: 25 additions & 31 deletions templates/render-overview.pug
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,45 +93,36 @@ html.update
else
span.deltaDown ▼
| from #{numFormat.format(data.last.DailyAcres)}
if data.current.TotalIncidentPersonnel
tr.important
th
span.current #{numFormat.format(data.current.TotalIncidentPersonnel)}
td
| Personnel
if !data.isNew && 'TotalIncidentPersonnel' in data.diff && data.last.TotalIncidentPersonnel
span.previous
if data.current.TotalIncidentPersonnel > data.last.TotalIncidentPersonnel
span.deltaUp ▲
else
span.deltaDown ▼
| from #{numFormat.format(data.last.TotalIncidentPersonnel)}
if data.current.EstimatedCostToDate
tr.important
th
span.current #{currencyFormat.format(data.current.EstimatedCostToDate)}
td
| Est. Cost
if !data.isNew && 'EstimatedCostToDate' in data.diff && data.last.EstimatedCostToDate
span.previous
if data.current.EstimatedCostToDate > data.last.EstimatedCostToDate
span.deltaUp ▲
else
span.deltaDown ▼
| from #{currencyFormat.format(data.last.EstimatedCostToDate)}

tr
td.details(colspan=2)
div.detailsRow
if data.current.Damage
- const dmg = data.current.Damage;
.damageData
if dmg.injuries || dmg.fatalities
.damageHuman
if dmg.fatalities
span.detail #{numFormat.format(dmg.fatalities)} Dead
if dmg.injuries
span.detail #{numFormat.format(dmg.injuries)} Injured
if dmg.structuresDestroyed || dmg.structuresDamaged
.damageBldg
if dmg.structuresDestroyed
span.detail #{numFormat.format(dmg.structuresDestroyed)} Bldgs Destroyed
if dmg.structuresDamaged
span.detail #{numFormat.format(dmg.structuresDamaged)} Damaged
if data.cities && (data.cities.closest || data.cities.biggest)
if data.cities.closest
span.detail #{data.cities.closest.directions}
br
span.detail
| #{data.cities.closest.directions}
if data.cities.biggest
span.detail #{data.cities.biggest.directions}
br
br
if data.current.FireCause
span.detail #{data.current.FireCause} Cause
span.detail
| #{data.current.FireCause} Cause
if data.current.FireDiscoveryDateTime
| -
if data.current.FireDiscoveryDateTime
span.detail Discovered #{friendlyDate(data.current.FireDiscoveryDateTime)}
if !data.isNew && data.last.ModifiedOnDateTime && data.current.ModifiedOnDateTime != data.last.ModifiedOnDateTime
Expand All @@ -140,6 +131,9 @@ html.update
if data.current.PerimDateTime
br
span.detail Perimeter Updated #{friendlyDate(data.current.PerimDateTime)}
if data.current.TotalIncidentPersonnel
br
span.detail #{numFormat.format(data.current.TotalIncidentPersonnel)} Personnel
if data.current.Notes
br
span.detail #{data.current.Notes}
Expand Down
15 changes: 14 additions & 1 deletion templates/style.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ table {
width: 100%;
}

.important {
.important, .damageData {
font-size: 125%;
}

Expand Down Expand Up @@ -298,6 +298,19 @@ table {
font-weight: normal;
}

.damageData {
padding-bottom: 5px;
}

.damageHuman {
color: white;
background-color: red;
}

.damageBldg {
background-color: orange;
}

.innerWarn {
padding: 5px;
}
Expand Down
Loading

0 comments on commit b0d8deb

Please sign in to comment.