Merge branch 'release/0.7.8'

Datavault-UK · Oct 25, 2021 · 9f7aec9 · 9f7aec9
2 parents c8f3f2d + f75b38b
commit 9f7aec9
Show file tree

Hide file tree

Showing 20 changed files with 222 additions and 186 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="https://user-images.githubusercontent.com/25080503/65772647-89525700-e132-11e9-80ff-12ad30a25466.png">
+  <img src="https://user-images.githubusercontent.com/25080503/65772647-89525700-e132-11e9-80ff-12ad30a25466.png" alt="dbtvault">
 </p>
 
 <p align="center">

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,10 +1,10 @@
-name: 'dbtvault'
-version: '0.7.7'
-require-dbt-version: [">=0.18.0", "<0.21.0"]
+name: dbtvault
+version: 0.7.8
+require-dbt-version: [">=0.20.0", "<0.22.0"]
 config-version: 2
 
 source-paths: ["models"]
-analysis-paths: ["analysis"] 
+analysis-paths: ["analysis"]
 test-paths: ["tests"]
 data-paths: ["data"]
 macro-paths: ["macros"]
@@ -16,4 +16,5 @@ clean-targets:
   - "dbt_modules"
 
 vars:
-  hash: MD5
+  hash: MD5
+  max_datetime: '{{ dbtvault.max_datetime() }}'
diff --git a/macros/internal/as_constant.sql b/macros/internal/as_constant.sql
@@ -6,7 +6,7 @@
 
 {%- macro default__as_constant(column_str) -%}
 
-    {% if column_str is not none and column_str is string and column_str %}
+    {%- if column_str is not none and column_str is string and column_str -%}
 
         {%- if column_str | first == "!" -%}
 

diff --git a/macros/internal/expand_column_list.sql b/macros/internal/expand_column_list.sql
@@ -10,6 +10,8 @@
 
 {%- if dbtvault.is_list(columns) -%}
 
+    {%- set columns = columns | reject("none") %}
+
     {%- for col in columns -%}
 
         {%- if col is string -%}

diff --git a/macros/internal/get_package_namespaces.sql b/macros/internal/get_package_namespaces.sql
diff --git a/macros/materialisations/period_mat_helpers.sql b/macros/materialisations/period_mat_helpers.sql
@@ -23,13 +23,11 @@
              TO_DATE({{ timestamp_field }}) < DATE_TRUNC('{{ period }}', TO_DATE('{{ start_timestamp }}') + INTERVAL '{{ offset }} {{ period }}' + INTERVAL '1 {{ period }}'))
       AND (TO_DATE({{ timestamp_field }}) >= TO_DATE('{{ start_timestamp }}'))
     {%- endset -%}
-
     {%- set filtered_sql = core_sql | replace("__PERIOD_FILTER__", period_filter) -%}
 
     {% do return(filtered_sql) %}
 {% endmacro %}
 
-
 {#-- GET_PERIOD_FILTER_SQL #}
 
 {%- macro get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%}
@@ -57,7 +55,6 @@
     select {{ target_cols_csv }} from ({{ filtered_sql.sql }})
 {%- endmacro %}
 
-
 {#-- GET_PERIOD_BOUNDARIES #}
 
 {%- macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}
@@ -73,23 +70,25 @@
     {% do return(macro) %}
 {%- endmacro %}
 
+
+
 {% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}
 
     {% set period_boundary_sql -%}
-        with data as (
-            select
-                coalesce(max({{ timestamp_field }}), '{{ start_date }}')::timestamp as start_timestamp,
-                coalesce({{ dbt_utils.dateadd('millisecond', 86399999, "nullif('" ~ stop_date | lower ~ "','none')::timestamp") }},
-                         {{ dbt_utils.current_timestamp() }} ) as stop_timestamp
-            from {{ target_schema }}.{{ target_table }}
+        WITH period_data AS (
+            SELECT
+                COALESCE(MAX({{ timestamp_field }}), '{{ start_date }}')::TIMESTAMP AS start_timestamp,
+                COALESCE({{ dbt_utils.dateadd('millisecond', 86399999, "NULLIF('" ~ stop_date | lower ~ "','none')::TIMESTAMP") }},
+                         {{ dbtvault.current_timestamp() }} ) AS stop_timestamp
+            FROM {{ target_schema }}.{{ target_table }}
         )
-        select
+        SELECT
             start_timestamp,
             stop_timestamp,
             {{ dbt_utils.datediff('start_timestamp',
                                   'stop_timestamp',
-                                  period) }} + 1 as num_periods
-        from data
+                                  period) }} + 1 AS num_periods
+        FROM period_data
     {%- endset %}
 
     {% set period_boundaries_dict = dbt_utils.get_query_results_as_dict(period_boundary_sql) %}
@@ -114,10 +113,11 @@
     {% do return(macro) %}
 {%- endmacro %}
 
+
 {%- macro default__get_period_of_load(period, offset, start_timestamp) -%}
 
     {% set period_of_load_sql -%}
-        SELECT DATE_TRUNC('{{ period }}', DATEADD({{ period }}, {{ offset }}, TO_DATE('{{start_timestamp}}'))) AS period_of_load
+        SELECT DATE_TRUNC('{{ period }}', DATEADD({{ period }}, {{ offset }}, TO_DATE('{{ start_timestamp }}'))) AS period_of_load
     {%- endset %}
 
     {% set period_of_load_dict = dbt_utils.get_query_results_as_dict(period_of_load_sql) %}

diff --git a/macros/materialisations/vault_insert_by_period_materialization.sql b/macros/materialisations/vault_insert_by_period_materialization.sql
@@ -28,7 +28,6 @@
                                                                        start_stop_dates.stop_date,
                                                                        0, period) %}
         {% set build_sql = create_table_as(False, target_relation, filtered_sql) %}
-
         {% do to_drop.append(tmp_relation) %}
 
     {% elif existing_relation.is_view or full_refresh_mode %}
@@ -73,18 +72,18 @@
                                                                   period_boundaries.stop_timestamp, i) %}
 
             {% call statement() -%}
-                {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }}
+                {{ create_table_as(True, tmp_relation, tmp_table_sql) }}
             {%- endcall %}
 
             {{ adapter.expand_target_column_types(from_relation=tmp_relation,
                                                   to_relation=target_relation) }}
 
             {%- set insert_query_name = 'main-' ~ i -%}
             {% call statement(insert_query_name, fetch_result=True) -%}
-                insert into {{ target_relation }} ({{ target_cols_csv }})
+                INSERT INTO {{ target_relation }} ({{ target_cols_csv }})
                 (
-                    select {{ target_cols_csv }}
-                    from {{ tmp_relation.include(schema=True) }}
+                    SELECT {{ target_cols_csv }}
+                    FROM {{ tmp_relation.include(schema=True) }}
                 );
             {%- endcall %}
 

diff --git a/macros/materialisations/vault_insert_by_rank_materialization.sql b/macros/materialisations/vault_insert_by_rank_materialization.sql
@@ -23,7 +23,6 @@
     {% if existing_relation is none %}
 
         {% set filtered_sql = dbtvault.replace_placeholder_with_rank_filter(sql, rank_column, 1) %}
-
         {% set build_sql = create_table_as(False, target_relation, filtered_sql) %}
 
         {% do to_drop.append(tmp_relation) %}
@@ -57,19 +56,22 @@
 
             {% set tmp_relation = make_temp_relation(this) %}
 
+            {# This call statement drops and then creates a temporary table #}
+            {# but MSSQL will fail to drop any temporary table created by a previous loop iteration #}
+            {# See MSSQL note and drop code below #}
             {% call statement() -%}
-                {{ dbt.create_table_as(True, tmp_relation, filtered_sql) }}
+                {{ create_table_as(True, tmp_relation, filtered_sql) }}
             {%- endcall %}
 
             {{ adapter.expand_target_column_types(from_relation=tmp_relation,
                                                   to_relation=target_relation) }}
 
             {%- set insert_query_name = 'main-' ~ i -%}
             {% call statement(insert_query_name, fetch_result=True) -%}
-                insert into {{ target_relation }} ({{ target_cols_csv }})
+                INSERT INTO {{ target_relation }} ({{ target_cols_csv }})
                 (
-                    select {{ target_cols_csv }}
-                    from {{ tmp_relation.include(schema=True) }}
+                    SELECT {{ target_cols_csv }}
+                    FROM {{ tmp_relation.include(schema=True) }}
                 );
             {%- endcall %}
 
@@ -89,12 +91,10 @@
                                                                                           rows_inserted,
                                                                                           model.unique_id)) }}
 
-
             {% do to_drop.append(tmp_relation) %}
             {% do adapter.commit() %}
 
         {% endfor %}
-
         {% call noop_statement('main', "INSERT {}".format(loop_vars['sum_rows_inserted']) ) -%}
             {{ filtered_sql }}
         {%- endcall %}

diff --git a/macros/staging/rank_columns.sql b/macros/staging/rank_columns.sql
@@ -12,19 +12,57 @@
 
         {%- if columns[col] is mapping and columns[col].partition_by and columns[col].order_by -%}
 
-            {%- if dbtvault.is_list(columns[col].order_by) -%}
-                {%- set order_by_str = columns[col].order_by | join(", ") -%}
+            {%- set order_by = columns[col].order_by -%}
+            {%- set partition_by = columns[col].partition_by -%}
+            {%- set dense_rank = columns[col].dense_rank -%}
+
+            {%- if dbtvault.is_nothing(dense_rank) %}
+                {%- set rank_type = "RANK()" -%}
+            {%- elif dense_rank is true -%}
+                {%- set rank_type = "DENSE_RANK()" -%}
+            {%- else -%}
+                {%- if execute -%}
+                    {%- do exceptions.raise_compiler_error('If dense_rank is provided, it must be true or false, not {}'.format(dense_rank)) -%}
+                {% endif %}
+            {%- endif -%}
+
+            {%- if dbtvault.is_list(order_by) -%}
+
+                {%- set order_by_str_lst = [] -%}
+
+                {% for order_by_col in order_by %}
+
+                    {%- if order_by_col is mapping %}
+                        {%- set column_name, direction = order_by_col.items()|first -%}
+                        {%- set order_by_str = "{} {}".format(column_name, direction) | trim -%}
+                    {%- else -%}
+                        {%- set order_by_str = order_by_col -%}
+                    {%- endif -%}
+
+                    {%- do order_by_str_lst.append(order_by_str) -%}
+                {%- endfor -%}
+
+                {%- set order_by_str = order_by_str_lst | join(", ") -%}
+
             {%- else -%}
-                {%- set order_by_str = columns[col].order_by -%}
+
+                {%- if order_by is mapping %}
+                    {%- set column_name, direction = order_by.items()|first -%}
+                {%- else -%}
+                    {%- set column_name = order_by -%}
+                    {%- set direction = '' -%}
+                {%- endif -%}
+
+                {%- set order_by_str = "{} {}".format(column_name, direction) | trim -%}
             {%- endif -%}
 
-            {%- if dbtvault.is_list(columns[col].partition_by) -%}
-                {%- set partition_by_str = columns[col].partition_by | join(", ") -%}
+            {%- if dbtvault.is_list(partition_by) -%}
+                {%- set partition_by_str = partition_by | join(", ") -%}
             {%- else -%}
-                {%- set partition_by_str = columns[col].partition_by -%}
+                {%- set partition_by_str = partition_by -%}
             {%- endif -%}
 
-            {{- "RANK() OVER (PARTITION BY {} ORDER BY {}) AS {}".format(partition_by_str, order_by_str, col) | indent(4) -}}
+            {{- "{} OVER (PARTITION BY {} ORDER BY {}) AS {}".format(rank_type, partition_by_str, order_by_str, col) | indent(4) -}}
 
         {%- endif -%}
 

diff --git a/macros/staging/stage.sql b/macros/staging/stage.sql
@@ -19,13 +19,13 @@
 
     {%- set error_message -%}
     Staging error: Missing source_model configuration. A source model name must be provided.
-    e.g. 
+    e.g.
     [REF STYLE]
     source_model: model_name
     OR
     [SOURCES STYLE]
     source_model:
-        source_name: source_table_name"
+        source_name: source_table_name
     {%- endset -%}
 
     {{- exceptions.raise_compiler_error(error_message) -}}