Skip to content

Commit dab345b

Browse files
smnorrissmnorris
authored
stream order - add parent/max and a function for filtering MVT (#123)
* fix #122 * parent table no longer exists * uncomment, drop temp stream order tables * remove obsolete refs * add the files * remove parent order from docs script * add function to server streams as MVT, filtering on max order of blueline, based on zoom level * add missing file * note new function in changes * document new function
1 parent 4feb183 commit dab345b

12 files changed

+164
-65
lines changed

‎CHANGES.txt

+6
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@ Changes
33

44
All issue numbers are relative to https://github.com/smnorris/fwapg/issues
55

6+
0.3.1 ()
7+
------------------
8+
- ensure fixes are applied for partial loads (#120)
9+
- add stream_order_parent and stream_order_max to output table and remove parent order lookup (#122)
10+
- add function postgisftw.fwa_streamsasmvt() - zoom dependent filtering of streams as MVT
11+
612
0.3.0 (2022-10-14)
713
------------------
814
- modify load process to use latest `bcdata`, and also avoid problems with dependent views (#117)

‎Makefile

+12-18
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ ALL_TARGETS = .make/db \
3636
.make/wbdhu12 \
3737
.make/hydrosheds \
3838
$(VALUE_ADDED_TARGETS) \
39-
.make/fwa_stream_order_parent \
4039
.make/fwa_streams_watersheds_lut \
4140
.make/fwa_functions \
4241
.make/fwa_waterbodies_upstream_area \
@@ -121,9 +120,20 @@ $(WSD_TARGETS): .make/spatial_large_load
121120

122121
# copy data from fwapg to whse_basemapping
123122
.make/fwa_stream_networks_sp: $(STREAM_TARGETS)
123+
# build the lookups
124+
$(PSQL) -f sql/tables/temp/fwa_stream_order_max.sql
125+
$(PSQL) -c "drop table if exists fwapg.fwa_stream_order_parent"
126+
$(PSQL) -c "create table fwapg.fwa_stream_order_parent \
127+
(blue_line_key integer primary key, stream_order_parent integer);"
128+
for wsg in $(GROUPS) ; do \
129+
$(PSQL) -v wsg=$$wsg -f sql/tables/temp/fwa_stream_order_parent.sql ; \
130+
done
131+
# load data per group so inserts are in managable chunks
124132
for wsg in $(GROUPS) ; do \
125133
set -e ; $(PSQL) -f sql/tables/spatial/large/fwa_stream_networks_sp.sql -v wsg=$$wsg ; \
126134
done
135+
$(PSQL) -c "drop table fwapg.fwa_stream_order_parent"
136+
$(PSQL) -c "drop table fwapg.fwa_stream_order_max"
127137
$(PSQL) -c "drop table fwapg.fwa_stream_networks_sp"
128138
$(PSQL) -c "vacuum analyze whse_basemapping.fwa_stream_networks_sp"
129139
# apply data fixes
@@ -215,22 +225,6 @@ data/FWA_BC.gdb.zip:
215225
$(PSQL) -c "COMMENT ON COLUMN whse_basemapping.fwa_streams_watersheds_lut.watershed_feature_id IS 'FWA fundamental watershed unique identifer';"
216226
touch $@
217227

218-
# create a table holding the parent stream order of all streams (where possible)
219-
.make/fwa_stream_order_parent: sql/tables/value_added_chunked/fwa_stream_order_parent.sql .make/fwa_stream_networks_sp
220-
# create table
221-
$(PSQL) -c "drop table if exists whse_basemapping.fwa_stream_order_parent"
222-
$(PSQL) -c "create table whse_basemapping.fwa_stream_order_parent \
223-
(blue_line_key integer primary key, stream_order_parent integer);"
224-
# load data per group so inserts are in managable chunks
225-
for wsg in $(GROUPS) ; do \
226-
$(PSQL) -v wsg=$$wsg -f $< ; \
227-
done
228-
# comment and index after load
229-
$(PSQL) -c "COMMENT ON TABLE whse_basemapping.fwa_stream_order_parent IS 'Streams (as blue_line_key) and the stream order of the stream they flow into';"
230-
$(PSQL) -c "COMMENT ON COLUMN whse_basemapping.fwa_stream_order_parent.blue_line_key IS 'FWA blue_line_key';"
231-
$(PSQL) -c "COMMENT ON COLUMN whse_basemapping.fwa_stream_order_parent.stream_order_parent IS 'The stream_order of the stream the blue_line_key flows into';"
232-
touch $@
233-
234228
# USA (lower 48) watersheds - USGS HU12 polygons
235229
# note that this is possbile to download via /vsizip/vsicurl but a direct download seems faster
236230
data/WBD_National_GDB.zip:
@@ -303,7 +297,7 @@ data/WBD_National_GDB.zip:
303297
touch $@
304298

305299
# additional FWA functions
306-
.make/fwa_functions: $(SPATIAL_BASIC) $(SPATIAL_LARGE) $(NON_SPATIAL_TARGETS) $(VALUE_ADDED_TARGETS) .make/fwa_streams_watersheds_lut .make/fwa_stream_order_parent \
300+
.make/fwa_functions: $(SPATIAL_BASIC) $(SPATIAL_LARGE) $(NON_SPATIAL_TARGETS) $(VALUE_ADDED_TARGETS) .make/fwa_streams_watersheds_lut \
307301
.make/hydrosheds \
308302
.make/wbdhu12
309303
$(PSQL) -f sql/functions/FWA_SliceWatershedAtPoint.sql

‎docs/03_tables.md

-11
Original file line numberDiff line numberDiff line change
@@ -75,17 +75,6 @@ A convenience lookup for quickly relating streams and fundamental watersheds
7575
| `linear_feature_id` | `bigint` | FWA stream segment unique identifier |
7676
| `watershed_feature_id` | `integer` | FWA fundamental watershed unique identifer |
7777

78-
## fwa_stream_order_parent
79-
80-
A convenince table, holding parent `stream_order` values for each stream in BC (as defined by `blue_line_key`).
81-
'Parent' stream order is the order of the `blue_line_key` stream into which the given stream drains.
82-
For side channels, `stream_order_parent` is the order of the main channel at the location of the downstream tributary with the side channel.
83-
84-
| Column | Type | Description |
85-
|--------|------|-------------|
86-
| `blue_line_key` | `integer` | See FWA documentation for blue_line_key description |
87-
| `stream_order_parent` | `integer` | The stream_order of the stream the blue_line_key flows into |
88-
8978

9079
## fwa_waterbodies
9180

‎docs/04_functions.md

+24
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,30 @@ Make the same request as the example above, but
309309
[at 10km](https://features.hillcrestgeo.ca/fwa/functions/fwa_locatealonginterval/items.html?blue_line_key=359572348&start_measure=1597489&interval_length=10000&end_measure=1706733&limit=100)
310310

311311

312+
## FWA_StreamsAsMVT
313+
314+
### Synopsis
315+
316+
```sql
317+
FWA_StreamsAsMVT(
318+
z integer,
319+
x integer,
320+
y integer
321+
)
322+
```
323+
324+
### Description
325+
326+
Return FWA streams as MVT, filtering by `stream_order_max` (the maximum order of a given stream / `blue_line_key`) based on the provided zoom level.
327+
Enables fast display of source 1:20,000 streams at all zoom levels (show full length of higher order streams at lower zooms), using `pg_tileserv`.
328+
329+
Note: for databases with modest resources, tiles at low zoom levels may be slow to render. The function should still be speedy enough for general use if [pg_tileserv is behind a cache](https://github.com/CrunchyData/pg_tileserv#basic-operation).
330+
331+
### Web service
332+
333+
[FWA_StreamsAsMVT](https://tiles.hillcrestgeo.ca/bcfishpass/postgisftw.fwa_streamsasmvt.html)
334+
335+
312336
## FWA_Upstream
313337

314338
### Synopsis

‎docs/table_reference.sh

-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ for table in approx_borders \
1111
bcboundary \
1212
named_streams \
1313
streams_watersheds_lut \
14-
stream_order_parent \
1514
waterbodies \
1615
waterbodies_upstream_area \
1716
watersheds_upstream_area \

‎sql/functions/FWA_StreamsAsMVT.sql

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
CREATE OR REPLACE
2+
FUNCTION postgisftw.fwa_streamsasmvt(
3+
z integer, x integer, y integer)
4+
RETURNS bytea
5+
AS $$
6+
DECLARE
7+
result bytea;
8+
BEGIN
9+
WITH
10+
11+
bounds AS (
12+
SELECT ST_TileEnvelope(z, x, y) AS geom
13+
),
14+
15+
mvtgeom AS (
16+
SELECT
17+
blue_line_key,
18+
gnis_name,
19+
stream_order_max,
20+
ST_AsMVTGeom(ST_Transform(ST_Force2D(s.geom), 3857), bounds.geom)
21+
FROM whse_basemapping.fwa_stream_networks_sp s, bounds
22+
WHERE ST_Intersects(s.geom, ST_Transform((select geom from bounds), 3005))
23+
AND s.edge_type != 6010
24+
AND wscode_ltree is not null
25+
AND stream_order_max >= (-z + 13)
26+
)
27+
28+
SELECT ST_AsMVT(mvtgeom, 'default')
29+
INTO result
30+
FROM mvtgeom;
31+
32+
RETURN result;
33+
END;
34+
$$
35+
LANGUAGE 'plpgsql'
36+
STABLE
37+
PARALLEL SAFE;
38+
39+
COMMENT ON FUNCTION postgisftw.fwa_streamsasmvt IS 'Zoom-level dependent FWA streams';

‎sql/misc/drop_all.sql

-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ drop table if exists whse_basemapping.fwa_basins_poly;
2929
drop table if exists whse_basemapping.fwa_bcboundary;
3030
drop table if exists whse_basemapping.fwa_named_streams;
3131
drop table if exists whse_basemapping.fwa_waterbodies;
32-
drop table if exists whse_basemapping.fwa_stream_order_parent;
3332

3433
drop table if exists whse_basemapping.fwa_assessment_watersheds_lut;
3534
drop table if exists whse_basemapping.fwa_assessment_watersheds_streams_lut;

‎sql/misc/fixes_fwa_stream_networks_sp.sql

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
-- Birkenhead River, coding problem due to distributary
2+
UPDATE whse_basemapping.fwa_stream_networks_sp
3+
SET local_watershed_code = fwa_watershed_code
4+
WHERE linear_feature_id IN (701704850,701705120,701705172,701705231,701704801,701705079,701705158,701704795,701704635,701705099);
5+
6+
-- Scar Creek
7+
UPDATE whse_basemapping.fwa_stream_networks_sp
8+
SET local_watershed_code = fwa_watershed_code
9+
WHERE linear_feature_id = 66063417;
10+
11+
-- Kaouk River
12+
UPDATE whse_basemapping.fwa_stream_networks_sp
13+
SET local_watershed_code = '930-665189-814505-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000'
14+
WHERE linear_feature_id IN (710531051, 710530976, 710530984);
15+
16+
UPDATE whse_basemapping.fwa_stream_networks_sp
17+
SET local_watershed_code = '930-665189-862690-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000'
18+
WHERE linear_feature_id = 710530993;
19+
20+
-- Memekay
21+
UPDATE whse_basemapping.fwa_stream_networks_sp
22+
SET local_watershed_code = '920-722273-347291-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000-000000'
23+
WHERE linear_feature_id IN (710445801, 710446876);

‎sql/tables/spatial/large/fwa_stream_networks_sp.sql

+33-28
Original file line numberDiff line numberDiff line change
@@ -3,36 +3,41 @@ delete from whse_basemapping.fwa_stream_networks_sp where watershed_group_code =
33
insert into whse_basemapping.fwa_stream_networks_sp (linear_feature_id, watershed_group_id,
44
edge_type, blue_line_key, watershed_key, fwa_watershed_code, local_watershed_code,
55
watershed_group_code, downstream_route_measure, length_metre, feature_source, gnis_id, gnis_name,
6-
left_right_tributary, stream_order, stream_magnitude, waterbody_key, blue_line_key_50k,
7-
watershed_code_50k, watershed_key_50k, watershed_group_code_50k, feature_code, geom)
6+
left_right_tributary, stream_order, stream_order_parent, stream_order_max, stream_magnitude,
7+
waterbody_key, blue_line_key_50k, watershed_code_50k, watershed_key_50k, watershed_group_code_50k,
8+
feature_code, geom)
89
select
9-
linear_feature_id::bigint as linear_feature_id,
10-
watershed_group_id::integer as watershed_group_id,
11-
edge_type::integer as edge_type,
12-
blue_line_key::integer as blue_line_key,
13-
watershed_key::integer as watershed_key,
14-
fwa_watershed_code,
15-
local_watershed_code,
16-
watershed_group_code,
17-
downstream_route_measure::double precision as downstream_route_measure,
18-
length_metre::double precision as length_metre,
19-
feature_source,
20-
gnis_id::integer as gnis_id,
21-
gnis_name,
22-
left_right_tributary,
23-
stream_order::integer as stream_order,
24-
stream_magnitude::integer as stream_magnitude,
25-
waterbody_key::integer as waterbody_key,
26-
blue_line_key_50k::integer as blue_line_key_50k,
27-
watershed_code_50k as watershed_code_50k,
28-
watershed_key_50k::integer as watershed_key_50k,
29-
watershed_group_code_50k,
30-
feature_code,
10+
s.linear_feature_id::bigint as linear_feature_id,
11+
s.watershed_group_id::integer as watershed_group_id,
12+
s.edge_type::integer as edge_type,
13+
s.blue_line_key::integer as blue_line_key,
14+
s.watershed_key::integer as watershed_key,
15+
s.fwa_watershed_code,
16+
s.local_watershed_code,
17+
s.watershed_group_code,
18+
s.downstream_route_measure::double precision as downstream_route_measure,
19+
s.length_metre::double precision as length_metre,
20+
s.feature_source,
21+
s.gnis_id::integer as gnis_id,
22+
s.gnis_name,
23+
s.left_right_tributary,
24+
s.stream_order::integer as stream_order,
25+
p.stream_order_parent as stream_order_parent,
26+
m.stream_order_max as stream_order_max,
27+
s.stream_magnitude::integer as stream_magnitude,
28+
s.waterbody_key::integer as waterbody_key,
29+
s.blue_line_key_50k::integer as blue_line_key_50k,
30+
s.watershed_code_50k as watershed_code_50k,
31+
s.watershed_key_50k::integer as watershed_key_50k,
32+
s.watershed_group_code_50k,
33+
s.feature_code,
3134
st_addmeasure(
32-
(st_dump(geom)).geom,
33-
downstream_route_measure::double precision,
34-
downstream_route_measure::double precision + st_length((st_dump(geom)).geom)
35+
(st_dump(s.geom)).geom,
36+
s.downstream_route_measure::double precision,
37+
s.downstream_route_measure::double precision + st_length((st_dump(s.geom)).geom)
3538
) as geom
36-
from fwapg.fwa_stream_networks_sp
39+
from fwapg.fwa_stream_networks_sp s
40+
left outer join fwapg.fwa_stream_order_parent p on s.blue_line_key = p.blue_line_key
41+
left outer join fwapg.fwa_stream_order_max m on s.blue_line_key = m.blue_line_key
3742
where watershed_group_code = :'wsg'
3843
order by random(); --https://blog.crunchydata.com/blog/tricks-for-faster-spatial-indexes

‎sql/tables/spatial/schema.sql

+2
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,8 @@ create table if not exists whse_basemapping.fwa_stream_networks_sp (
385385
gnis_name character varying(80),
386386
left_right_tributary character varying(7),
387387
stream_order integer,
388+
stream_order_parent integer,
389+
stream_order_max integer,
388390
stream_magnitude integer,
389391
waterbody_key integer,
390392
blue_line_key_50k integer,

‎sql/tables/temp/fwa_stream_order_max.sql

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-- report on max order of given blue line key, useful for filtering when mapping at various scales
2+
3+
drop table if exists fwapg.fwa_stream_order_max ;
4+
5+
create table fwapg.fwa_stream_order_max
6+
(blue_line_key integer primary key,
7+
stream_order_max integer);
8+
9+
insert into fwapg.fwa_stream_order_max
10+
(blue_line_key, stream_order_max)
11+
select
12+
blue_line_key,
13+
max(stream_order) as max_stream_order
14+
from fwapg.fwa_stream_networks_sp
15+
where wscode_ltree is not null
16+
and watershed_key = blue_line_key
17+
and edge_type != 6010
18+
and wscode_ltree <@ '999'::ltree is false
19+
group by blue_line_key;

‎sql/tables/value_added_chunked/fwa_stream_order_parent.sql ‎sql/tables/temp/fwa_stream_order_parent.sql

+6-6
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@
22
-- (generally used for finding small channels that drain into major rivers)
33

44
-- load primary channels
5-
insert into whse_basemapping.fwa_stream_order_parent
5+
insert into fwapg.fwa_stream_order_parent
66
(blue_line_key, stream_order_parent)
77
select distinct on (a.blue_line_key)
88
a.blue_line_key,
99
b.stream_order as stream_order_parent
10-
from whse_basemapping.fwa_stream_networks_sp a
11-
left outer join whse_basemapping.fwa_stream_networks_sp b
10+
from fwapg.fwa_stream_networks_sp a
11+
left outer join fwapg.fwa_stream_networks_sp b
1212
on a.wscode_ltree = b.localcode_ltree
1313
where
1414
a.watershed_group_code = :'wsg'
@@ -26,15 +26,15 @@ insert into whse_basemapping.fwa_stream_order_parent
2626
-- as the side channel)
2727
-- NOTE - this will not populate parent order for side channels that do not
2828
-- have local codes
29-
insert into whse_basemapping.fwa_stream_order_parent (
29+
insert into fwapg.fwa_stream_order_parent (
3030
blue_line_key,
3131
stream_order_parent
3232
)
3333
select distinct on (a.blue_line_key)
3434
a.blue_line_key,
3535
b.stream_order as stream_order_parent
36-
from whse_basemapping.fwa_stream_networks_sp a
37-
left outer join whse_basemapping.fwa_stream_networks_sp b
36+
from fwapg.fwa_stream_networks_sp a
37+
left outer join fwapg.fwa_stream_networks_sp b
3838
on (a.wscode_ltree = b.wscode_ltree and
3939
a.localcode_ltree = b.localcode_ltree)
4040
where

0 commit comments

Comments
 (0)