Merge pull request #443 from dbt-labs/feature/remove-line-comments-SQL

Author: b-per

dbt_project.yml

@@ -87,3 +87,6 @@ vars:
   # -- Execution variables --
   insert_batch_size: "{{ 500 if target.type in ['athena', 'bigquery'] else 10000 }}"
   max_depth_dag: "{{ 9 if target.type in ['bigquery', 'spark', 'databricks'] else 4 if target.type in ['athena', 'trino'] else -1 }}"
+
+  # -- Code complexity variables --
+  comment_chars: ["--"]

integration_tests/models/marts/fct_model_6.sql

@@ -5,86 +5,89 @@
 }}
 
 select 1 as id 
--- from {{ ref('stg_model_3') }}
 
--- union all 
--- select
---     3 as id
--- from my_db.my_schema.my_table
--- union all 
--- select
---     3 as id
--- from 'my_db'.'my_schema'.'my_table'
--- union all 
--- select
---     3 as id
--- from "my_db"."my_schema"."my_table"
--- union all 
--- select
---     3 as id
--- from `my_db`.`my_schema`.`my_table`
--- union all 
--- select
---     3 as id
--- from [my_db].[my_schema].[my_table]
+-- depends on {{ ref('stg_model_3') }} 
 
--- union all
--- select 
---     4 as id
--- from my_schema.raw_relation_5
--- union all
--- select 
---     4 as id
--- from 'my_schema'.'raw_relation_5' 
--- union all
--- select 
---     4 as id
--- from "my_schema"."raw_relation_5"
--- union all
--- select 
---     4 as id
--- from `my_schema`.`raw_relation_5`
--- union all
--- select 
---     4 as id
--- from [my_schema].[raw_relation_5] 
+{#
+  from {{ ref('stg_model_3') }}   
+  union all 
+  select
+      3 as id
+  from my_db.my_schema.my_table
+  union all 
+  select
+      3 as id
+  from 'my_db'.'my_schema'.'my_table'
+  union all 
+  select
+      3 as id
+  from "my_db"."my_schema"."my_table"
+  union all 
+  select
+      3 as id
+  from `my_db`.`my_schema`.`my_table`
+  union all 
+  select
+      3 as id
+  from [my_db].[my_schema].[my_table]   
+  union all
+  select 
+      4 as id
+  from my_schema.raw_relation_5
+  union all
+  select 
+      4 as id
+  from 'my_schema'.'raw_relation_5' 
+  union all
+  select 
+      4 as id
+  from "my_schema"."raw_relation_5"
+  union all
+  select 
+      4 as id
+  from `my_schema`.`raw_relation_5`
+  union all
+  select 
+      4 as id
+  from [my_schema].[raw_relation_5]    
+  union all
+  select 
+      4 as id
+  from `raw_relation_1` 
+  union all
+  select 
+      4 as id
+  from "raw_relation_2" 
+  union all
+  select 
+      4 as id
+  from [raw_relation_3]  
+  union all
+  select 
+      4 as id
+  from 'raw_relation_4'    
+  union all
+  select
+      4 as id
+  from {{ var("my_table_reference") }}
+  union all
+  select
+      4 as id
+  from {{ var('my_table_reference') }}   
+  union all
+  
+  -- the following is SQL commented so it should not be found as a hard coded reference
+--   select
+--       6 as id
+--   from my_db.my_schema.my_table222
+  
+  select
+      5 as id
+  from {{ var("my_table_reference", "table_d") }}
+  union all
+  select
+      5 as id
+  from {{ var('my_table_reference', 'table_d') }}
 
 
--- union all
--- select 
---     4 as id
--- from `raw_relation_1` 
--- union all
--- select 
---     4 as id
--- from "raw_relation_2" 
--- union all
--- select 
---     4 as id
--- from [raw_relation_3]  
--- union all
--- select 
---     4 as id
--- from 'raw_relation_4' 
-
--- union all
--- select
---     4 as id
--- from {{ var("my_table_reference") }}
--- union all
--- select
---     4 as id
--- from {{ var('my_table_reference') }}
-
-
--- union all
--- select
---     5 as id
--- from {{ var("my_table_reference", "table_d") }}
--- union all
--- select
---     5 as id
--- from {{ var('my_table_reference', 'table_d') }}
--- select
---     7 as id
--- from {{ var('my_table_reference', 'table_d') }}
+#}

macros/find_all_hard_coded_references.sql

@@ -14,6 +14,11 @@
         {%- set model_raw_sql = '' -%}
         {%- endif -%}
 
+        {# we remove the comments that start with -- , or other characters configured #}
+        {%- set re = modules.re -%}
+        {%- set comment_chars_match = "(" ~ var('comment_chars') | join("|") ~ ").*" -%}
+        {%- set model_raw_sql_no_comments = re.sub(comment_chars_match, '', model_raw_sql) -%}
+
         {#-
             REGEX Explanations
 
@@ -234,7 +239,7 @@
 
         {%- for regex_name, regex_pattern in from_hard_coded_references.items() -%}
 
-            {%- set all_regex_matches = re.findall(regex_pattern, model_raw_sql) -%}
+            {%- set all_regex_matches = re.findall(regex_pattern, model_raw_sql_no_comments) -%}
 
                 {%- for match in all_regex_matches -%}