Skip to content

Commit ff36cf1

Browse files
b-perb-per
authored
Merge branch 'main' into add-version-to-output
2 parents 6c2bfc1 + 33fb3a7 commit ff36cf1

File tree

7 files changed

+97
-104
lines changed

7 files changed

+97
-104
lines changed

‎dbt_project.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,6 @@ vars:
8787
# -- Execution variables --
8888
insert_batch_size: "{{ 500 if target.type in ['athena', 'bigquery'] else 10000 }}"
8989
max_depth_dag: "{{ 9 if target.type in ['bigquery', 'spark', 'databricks'] else 4 if target.type in ['athena', 'trino'] else -1 }}"
90+
91+
# -- Code complexity variables --
92+
comment_chars: ["--"]

‎integration_tests/dbt_project.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,5 @@ vars:
6767
new_model_type_folder_name: 'my_new_models'
6868
new_model_type_prefixes: 'nwmdl_'
6969
insert_batch_size: 100
70-
too_many_joins_threshold: 3
70+
too_many_joins_threshold: 3
71+
chained_views_threshold: 2

‎integration_tests/models/marts/fct_model_6.sql‎

Lines changed: 82 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -5,86 +5,89 @@
55
}}
66

77
select 1 as id
8-
-- from {{ ref('stg_model_3') }}
98

10-
-- union all
11-
-- select
12-
-- 3 as id
13-
-- from my_db.my_schema.my_table
14-
-- union all
15-
-- select
16-
-- 3 as id
17-
-- from 'my_db'.'my_schema'.'my_table'
18-
-- union all
19-
-- select
20-
-- 3 as id
21-
-- from "my_db"."my_schema"."my_table"
22-
-- union all
23-
-- select
24-
-- 3 as id
25-
-- from `my_db`.`my_schema`.`my_table`
26-
-- union all
27-
-- select
28-
-- 3 as id
29-
-- from [my_db].[my_schema].[my_table]
9+
-- depends on {{ ref('stg_model_3') }}
3010

31-
-- union all
32-
-- select
33-
-- 4 as id
34-
-- from my_schema.raw_relation_5
35-
-- union all
36-
-- select
37-
-- 4 as id
38-
-- from 'my_schema'.'raw_relation_5'
39-
-- union all
40-
-- select
41-
-- 4 as id
42-
-- from "my_schema"."raw_relation_5"
43-
-- union all
44-
-- select
45-
-- 4 as id
46-
-- from `my_schema`.`raw_relation_5`
47-
-- union all
48-
-- select
49-
-- 4 as id
50-
-- from [my_schema].[raw_relation_5]
11+
{#
12+
from {{ ref('stg_model_3') }}
13+
union all
14+
select
15+
3 as id
16+
from my_db.my_schema.my_table
17+
union all
18+
select
19+
3 as id
20+
from 'my_db'.'my_schema'.'my_table'
21+
union all
22+
select
23+
3 as id
24+
from "my_db"."my_schema"."my_table"
25+
union all
26+
select
27+
3 as id
28+
from `my_db`.`my_schema`.`my_table`
29+
union all
30+
select
31+
3 as id
32+
from [my_db].[my_schema].[my_table]
33+
union all
34+
select
35+
4 as id
36+
from my_schema.raw_relation_5
37+
union all
38+
select
39+
4 as id
40+
from 'my_schema'.'raw_relation_5'
41+
union all
42+
select
43+
4 as id
44+
from "my_schema"."raw_relation_5"
45+
union all
46+
select
47+
4 as id
48+
from `my_schema`.`raw_relation_5`
49+
union all
50+
select
51+
4 as id
52+
from [my_schema].[raw_relation_5]
53+
union all
54+
select
55+
4 as id
56+
from `raw_relation_1`
57+
union all
58+
select
59+
4 as id
60+
from "raw_relation_2"
61+
union all
62+
select
63+
4 as id
64+
from [raw_relation_3]
65+
union all
66+
select
67+
4 as id
68+
from 'raw_relation_4'
69+
union all
70+
select
71+
4 as id
72+
from {{ var("my_table_reference") }}
73+
union all
74+
select
75+
4 as id
76+
from {{ var('my_table_reference') }}
77+
union all
78+
79+
-- the following is SQL commented so it should not be found as a hard coded reference
80+
-- select
81+
-- 6 as id
82+
-- from my_db.my_schema.my_table222
83+
84+
select
85+
5 as id
86+
from {{ var("my_table_reference", "table_d") }}
87+
union all
88+
select
89+
5 as id
90+
from {{ var('my_table_reference', 'table_d') }}
5191

5292

53-
-- union all
54-
-- select
55-
-- 4 as id
56-
-- from `raw_relation_1`
57-
-- union all
58-
-- select
59-
-- 4 as id
60-
-- from "raw_relation_2"
61-
-- union all
62-
-- select
63-
-- 4 as id
64-
-- from [raw_relation_3]
65-
-- union all
66-
-- select
67-
-- 4 as id
68-
-- from 'raw_relation_4'
69-
70-
-- union all
71-
-- select
72-
-- 4 as id
73-
-- from {{ var("my_table_reference") }}
74-
-- union all
75-
-- select
76-
-- 4 as id
77-
-- from {{ var('my_table_reference') }}
78-
79-
80-
-- union all
81-
-- select
82-
-- 5 as id
83-
-- from {{ var("my_table_reference", "table_d") }}
84-
-- union all
85-
-- select
86-
-- 5 as id
87-
-- from {{ var('my_table_reference', 'table_d') }}
88-
-- select
89-
-- 7 as id
90-
-- from {{ var('my_table_reference', 'table_d') }}
93+
#}

‎integration_tests/seeds/performance/test_fct_chained_views_dependencies.csv‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,2 @@
11
parent,child,distance
22
stg_model_1,dim_model_7,3
3-
stg_model_1,int_model_5.v2,2
4-
stg_model_2,dim_model_7,2
5-
int_model_4,dim_model_7,2
6-
stg_model_1,dim_model_7,2
7-
stg_model_4,dim_model_7,1
8-
stg_model_2,stg_model_4,1
9-
int_model_4,int_model_5.v2,1
10-
int_model_5.v2,dim_model_7,1
11-
stg_model_1,int_model_5.v2,1
12-
stg_model_1,int_model_4,1
13-
fct_model_9,stg_model_5,1
14-
stg_model_3,fct_model_6,1
15-
stg_model_1,fct_model_10,1
16-
stg_model_2,fct_model_10,1
17-
stg_model_3,fct_model_10,1

‎macros/find_all_hard_coded_references.sql‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@
1414
{%- set model_raw_sql = '' -%}
1515
{%- endif -%}
1616

17+
{# we remove the comments that start with -- , or other characters configured #}
18+
{%- set re = modules.re -%}
19+
{%- set comment_chars_match = "(" ~ var('comment_chars') | join("|") ~ ").*" -%}
20+
{%- set model_raw_sql_no_comments = re.sub(comment_chars_match, '', model_raw_sql) -%}
21+
1722
{#-
1823
REGEX Explanations
1924

@@ -234,7 +239,7 @@
234239

235240
{%- for regex_name, regex_pattern in from_hard_coded_references.items() -%}
236241

237-
{%- set all_regex_matches = re.findall(regex_pattern, model_raw_sql) -%}
242+
{%- set all_regex_matches = re.findall(regex_pattern, model_raw_sql_no_comments) -%}
238243

239244
{%- for match in all_regex_matches -%}
240245

‎models/marts/performance/fct_chained_views_dependencies.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ final as (
1616
from all_relationships
1717
where is_dependent_on_chain_of_views
1818
and child_resource_type = 'model'
19+
and distance > {{ var('chained_views_threshold') }}
1920
)
2021

2122
select * from final

‎models/marts/performance/performance.yml‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,6 @@ models:
66
This returns models dependent on chains of "non-physically-materialized" models (views and ephemerals),
77
highlighting potential cases for improving performance by switching the materialization of model(s) within
88
the chain to table or incremental.
9-
columns:
10-
- name: distance
11-
tests:
12-
- dbt_utils.accepted_range:
13-
name: valid_chained_views_dependencies
14-
max_value: "{{ var('chained_views_threshold') }}"
15-
inclusive: false
16-
severity: warn
9+
tests:
10+
- is_empty:
11+
severity: warn

0 commit comments

Comments
 (0)