Merge branch 'main' into add-version-to-output

Author: b-per

dbt_project.yml

@@ -87,3 +87,6 @@ vars:
   # -- Execution variables --
   insert_batch_size: "{{ 500 if target.type in ['athena', 'bigquery'] else 10000 }}"
   max_depth_dag: "{{ 9 if target.type in ['bigquery', 'spark', 'databricks'] else 4 if target.type in ['athena', 'trino'] else -1 }}"
+
+  # -- Code complexity variables --
+  comment_chars: ["--"]

integration_tests/dbt_project.yml

@@ -67,4 +67,5 @@ vars:
   new_model_type_folder_name: 'my_new_models'
   new_model_type_prefixes: 'nwmdl_'
   insert_batch_size: 100
-  too_many_joins_threshold: 3
+  too_many_joins_threshold: 3
+  chained_views_threshold: 2

integration_tests/models/marts/fct_model_6.sql

@@ -5,86 +5,89 @@
 }}
 
 select 1 as id 
--- from {{ ref('stg_model_3') }}
 
--- union all 
--- select
---     3 as id
--- from my_db.my_schema.my_table
--- union all 
--- select
---     3 as id
--- from 'my_db'.'my_schema'.'my_table'
--- union all 
--- select
---     3 as id
--- from "my_db"."my_schema"."my_table"
--- union all 
--- select
---     3 as id
--- from `my_db`.`my_schema`.`my_table`
--- union all 
--- select
---     3 as id
--- from [my_db].[my_schema].[my_table]
+-- depends on {{ ref('stg_model_3') }} 
 
--- union all
--- select 
---     4 as id
--- from my_schema.raw_relation_5
--- union all
--- select 
---     4 as id
--- from 'my_schema'.'raw_relation_5' 
--- union all
--- select 
---     4 as id
--- from "my_schema"."raw_relation_5"
--- union all
--- select 
---     4 as id
--- from `my_schema`.`raw_relation_5`
--- union all
--- select 
---     4 as id
--- from [my_schema].[raw_relation_5] 
+{#
+  from {{ ref('stg_model_3') }}   
+  union all 
+  select
+      3 as id
+  from my_db.my_schema.my_table
+  union all 
+  select
+      3 as id
+  from 'my_db'.'my_schema'.'my_table'
+  union all 
+  select
+      3 as id
+  from "my_db"."my_schema"."my_table"
+  union all 
+  select
+      3 as id
+  from `my_db`.`my_schema`.`my_table`
+  union all 
+  select
+      3 as id
+  from [my_db].[my_schema].[my_table]   
+  union all
+  select 
+      4 as id
+  from my_schema.raw_relation_5
+  union all
+  select 
+      4 as id
+  from 'my_schema'.'raw_relation_5' 
+  union all
+  select 
+      4 as id
+  from "my_schema"."raw_relation_5"
+  union all
+  select 
+      4 as id
+  from `my_schema`.`raw_relation_5`
+  union all
+  select 
+      4 as id
+  from [my_schema].[raw_relation_5]    
+  union all
+  select 
+      4 as id
+  from `raw_relation_1` 
+  union all
+  select 
+      4 as id
+  from "raw_relation_2" 
+  union all
+  select 
+      4 as id
+  from [raw_relation_3]  
+  union all
+  select 
+      4 as id
+  from 'raw_relation_4'    
+  union all
+  select
+      4 as id
+  from {{ var("my_table_reference") }}
+  union all
+  select
+      4 as id
+  from {{ var('my_table_reference') }}   
+  union all
+  
+  -- the following is SQL commented so it should not be found as a hard coded reference
+--   select
+--       6 as id
+--   from my_db.my_schema.my_table222
+  
+  select
+      5 as id
+  from {{ var("my_table_reference", "table_d") }}
+  union all
+  select
+      5 as id
+  from {{ var('my_table_reference', 'table_d') }}
 
 
--- union all
--- select 
---     4 as id
--- from `raw_relation_1` 
--- union all
--- select 
---     4 as id
--- from "raw_relation_2" 
--- union all
--- select 
---     4 as id
--- from [raw_relation_3]  
--- union all
--- select 
---     4 as id
--- from 'raw_relation_4' 
-
--- union all
--- select
---     4 as id
--- from {{ var("my_table_reference") }}
--- union all
--- select
---     4 as id
--- from {{ var('my_table_reference') }}
-
-
--- union all
--- select
---     5 as id
--- from {{ var("my_table_reference", "table_d") }}
--- union all
--- select
---     5 as id
--- from {{ var('my_table_reference', 'table_d') }}
--- select
---     7 as id
--- from {{ var('my_table_reference', 'table_d') }}
+#}

integration_tests/seeds/performance/test_fct_chained_views_dependencies.csv

@@ -1,17 +1,2 @@
 parent,child,distance
 stg_model_1,dim_model_7,3
-stg_model_1,int_model_5.v2,2
-stg_model_2,dim_model_7,2
-int_model_4,dim_model_7,2
-stg_model_1,dim_model_7,2
-stg_model_4,dim_model_7,1
-stg_model_2,stg_model_4,1
-int_model_4,int_model_5.v2,1
-int_model_5.v2,dim_model_7,1
-stg_model_1,int_model_5.v2,1
-stg_model_1,int_model_4,1
-fct_model_9,stg_model_5,1
-stg_model_3,fct_model_6,1
-stg_model_1,fct_model_10,1
-stg_model_2,fct_model_10,1
-stg_model_3,fct_model_10,1

macros/find_all_hard_coded_references.sql

@@ -14,6 +14,11 @@
         {%- set model_raw_sql = '' -%}
         {%- endif -%}
 
+        {# we remove the comments that start with -- , or other characters configured #}
+        {%- set re = modules.re -%}
+        {%- set comment_chars_match = "(" ~ var('comment_chars') | join("|") ~ ").*" -%}
+        {%- set model_raw_sql_no_comments = re.sub(comment_chars_match, '', model_raw_sql) -%}
+
         {#-
             REGEX Explanations
 
@@ -234,7 +239,7 @@
 
         {%- for regex_name, regex_pattern in from_hard_coded_references.items() -%}
 
-            {%- set all_regex_matches = re.findall(regex_pattern, model_raw_sql) -%}
+            {%- set all_regex_matches = re.findall(regex_pattern, model_raw_sql_no_comments) -%}
 
                 {%- for match in all_regex_matches -%}
 

models/marts/performance/fct_chained_views_dependencies.sql

@@ -16,6 +16,7 @@ final as (
     from all_relationships
     where is_dependent_on_chain_of_views
     and child_resource_type = 'model'
+    and distance > {{ var('chained_views_threshold') }}
 )
 
 select * from final

models/marts/performance/performance.yml

@@ -6,11 +6,6 @@ models:
       This returns models dependent on chains of "non-physically-materialized" models (views and ephemerals),
       highlighting potential cases for improving performance by switching the materialization of model(s) within 
       the chain to table or incremental. 
-    columns:
-      - name: distance
-        tests:
-          - dbt_utils.accepted_range:
-              name: valid_chained_views_dependencies
-              max_value: "{{ var('chained_views_threshold') }}"
-              inclusive: false
-              severity: warn
+    tests:
+      - is_empty:
+          severity: warn