Skip to content

Commit 36c4beb

Browse files
charlie-costanzocharlie-costanzo
authored
NTD: staging and mart tables for 2022 agency data (#3605)
* add new tables to yml files * add new staging and mart tables for 2022 agency information * fix duplicate entries in yml file
1 parent 43afcce commit 36c4beb

File tree

4 files changed

+125
-4
lines changed

4 files changed

+125
-4
lines changed

‎warehouse/models/mart/ntd_annual_reporting/dim_2022_agency_information.sql

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
WITH staging_agency_information AS (
2+
SELECT *
3+
FROM {{ ref('stg_ntd__2022_agency_information') }}
4+
),
5+
6+
dim_2022_agency_information AS (
7+
SELECT *
8+
FROM staging_agency_information
9+
)
10+
11+
SELECT
12+
number_of_state_counties,
13+
tam_tier,
14+
personal_vehicles,
15+
uza_name,
16+
tribal_area_name,
17+
service_area_sq_miles,
18+
voms_do,
19+
url,
20+
region,
21+
state_admin_funds_expended,
22+
zip_code_ext,
23+
zip_code,
24+
ueid,
25+
address_line_2,
26+
number_of_counties_with_service,
27+
reporter_acronym,
28+
original_due_date,
29+
sq_miles,
30+
address_line_1,
31+
p_o__box,
32+
fy_end_date,
33+
service_area_pop,
34+
state,
35+
subrecipient_type,
36+
population,
37+
reporting_module,
38+
volunteer_drivers,
39+
doing_business_as,
40+
reporter_type,
41+
legacy_ntd_id,
42+
total_voms,
43+
primary_uza_uace_code,
44+
reported_by_name,
45+
fta_recipient_id,
46+
city,
47+
voms_pt,
48+
organization_type,
49+
agency_name,
50+
ntd_id,
51+
reported_by_ntd_id,
52+
density,
53+
state_parent_ntd_id,
54+
dt,
55+
execution_ts
56+
FROM dim_2022_agency_information

‎warehouse/models/staging/ntd_annual_reporting/_src.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ sources:
179179
database: "{{ env_var('DBT_SOURCE_DATABASE', var('SOURCE_DATABASE')) }}"
180180
schema: external_ntd__annual_reporting
181181
tables:
182+
- name: 2022__annual_database_agency_information
183+
- name: 2023__annual_database_agency_information
184+
- name: 2023__annual_database_contractual_relationships
182185
- name: multi_year__breakdowns
183186
- name: multi_year__breakdowns_by_agency
184187
- name: multi_year__capital_expenses_by_capital_use
@@ -501,5 +504,3 @@ sources:
501504
- name: multi_year__track_and_roadway_guideway_age_distribution
502505
- name: multi_year__vehicles_age_distribution
503506
- name: multi_year__vehicles_type_count_by_agency
504-
- name: 2023__annual_database_agency_information
505-
- name: 2023__annual_database_contractual_relationships

‎warehouse/models/staging/ntd_annual_reporting/_stg_ntd_annual_reporting.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,9 @@ x-common-fields:
174174
description: '{{ doc("ntd_execution_timestamp") }}'
175175

176176
models:
177+
- name: stg_ntd__2022_agency_information
178+
- name: stg_ntd__2023_agency_information
179+
- name: stg_ntd__2023_contractual_relationships
177180
- name: stg_ntd__breakdowns
178181
- name: stg_ntd__breakdowns_by_agency
179182
- name: stg_ntd__capital_expenses_by_capital_use
@@ -495,5 +498,3 @@ models:
495498
- name: stg_ntd__track_and_roadway_guideway_age_distribution
496499
- name: stg_ntd__vehicles_age_distribution
497500
- name: stg_ntd__vehicles_type_count_by_agency
498-
- name: stg_ntd__2023_agency_information
499-
- name: stg_ntd__2023_contractual_relationships

‎warehouse/models/staging/ntd_annual_reporting/stg_ntd__2022_agency_information.sql

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
WITH external_agency_information AS (
2+
SELECT *
3+
FROM {{ source('external_ntd__annual_reporting', '2022__annual_database_agency_information') }}
4+
),
5+
6+
get_latest_extract AS(
7+
SELECT *
8+
FROM external_agency_information
9+
-- we pull the whole table every month in the pipeline, so this gets only the latest extract
10+
QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
11+
),
12+
13+
stg_ntd__2022_agency_information AS (
14+
SELECT *
15+
FROM get_latest_extract
16+
)
17+
18+
SELECT
19+
SAFE_CAST(number_of_state_counties AS NUMERIC) AS number_of_state_counties,
20+
{{ trim_make_empty_string_null('tam_tier') }} AS tam_tier,
21+
SAFE_CAST(personal_vehicles AS NUMERIC) AS personal_vehicles,
22+
{{ trim_make_empty_string_null('uza_name') }} AS uza_name,
23+
{{ trim_make_empty_string_null('tribal_area_name') }} AS tribal_area_name,
24+
SAFE_CAST(service_area_sq_miles AS NUMERIC) AS service_area_sq_miles,
25+
SAFE_CAST(voms_do AS NUMERIC) AS voms_do,
26+
{{ trim_make_empty_string_null('url') }} AS url,
27+
SAFE_CAST(region AS INTEGER) AS region,
28+
SAFE_CAST(state_admin_funds_expended AS NUMERIC) AS state_admin_funds_expended,
29+
SAFE_CAST(zip_code_ext AS NUMERIC) AS zip_code_ext,
30+
SAFE_CAST(zip_code AS NUMERIC) AS zip_code,
31+
{{ trim_make_empty_string_null('ueid') }} AS ueid,
32+
{{ trim_make_empty_string_null('address_line_2') }} AS address_line_2,
33+
SAFE_CAST(number_of_counties_with_service AS NUMERIC) AS number_of_counties_with_service,
34+
{{ trim_make_empty_string_null('reporter_acronym') }} AS reporter_acronym,
35+
SAFE_CAST(original_due_date AS INTEGER) AS original_due_date,
36+
SAFE_CAST(sq_miles AS NUMERIC) AS sq_miles,
37+
{{ trim_make_empty_string_null('address_line_1') }} AS address_line_1,
38+
{{ trim_make_empty_string_null('p_o__box') }} AS p_o__box,
39+
SAFE_CAST(fy_end_date AS INTEGER) AS fy_end_date,
40+
SAFE_CAST(service_area_pop AS NUMERIC) AS service_area_pop,
41+
{{ trim_make_empty_string_null('state') }} AS state,
42+
{{ trim_make_empty_string_null('subrecipient_type') }} AS subrecipient_type,
43+
SAFE_CAST(population AS NUMERIC) AS population,
44+
{{ trim_make_empty_string_null('reporting_module') }} AS reporting_module,
45+
SAFE_CAST(volunteer_drivers AS NUMERIC) AS volunteer_drivers,
46+
{{ trim_make_empty_string_null('doing_business_as') }} AS doing_business_as,
47+
{{ trim_make_empty_string_null('reporter_type') }} AS reporter_type,
48+
{{ trim_make_empty_string_null('legacy_ntd_id') }} AS legacy_ntd_id,
49+
SAFE_CAST(total_voms AS INTEGER) AS total_voms,
50+
SAFE_CAST(primary_uza_uace_code AS NUMERIC) AS primary_uza_uace_code,
51+
{{ trim_make_empty_string_null('reported_by_name') }} AS reported_by_name,
52+
SAFE_CAST(fta_recipient_id AS NUMERIC) AS fta_recipient_id,
53+
{{ trim_make_empty_string_null('city') }} AS city,
54+
SAFE_CAST(voms_pt AS NUMERIC) AS voms_pt,
55+
{{ trim_make_empty_string_null('organization_type') }} AS organization_type,
56+
{{ trim_make_empty_string_null('agency_name') }} AS agency_name,
57+
{{ trim_make_empty_string_null('ntd_id') }} AS ntd_id,
58+
{{ trim_make_empty_string_null('reported_by_ntd_id') }} AS reported_by_ntd_id,
59+
SAFE_CAST(density AS NUMERIC) AS density,
60+
{{ trim_make_empty_string_null('state_parent_ntd_id') }} AS state_parent_ntd_id,
61+
dt,
62+
execution_ts
63+
FROM stg_ntd__2022_agency_information

0 commit comments

Comments
 (0)