init

dbt-bigquery/.changes/unreleased/Features-20250207-090315.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Leverage static partitioning in dbt-bigquery materialization strategy
+time: 2025-02-07T09:03:15.963098Z
+custom:
+    Author: borjav
+    Issue: "538"

dbt-bigquery/src/dbt/include/bigquery/macros/adapters/datetime.sql

@@ -0,0 +1,27 @@
+{% macro generate_dates_in_range(start_date_datetime, end_date_datetime, granularity) %}
+  {#-- Generate a list of datetimes between two dates #}
+  {% set total_offset_seconds = (end_date_datetime - start_date_datetime).total_seconds()  %}
+  {% set total_offset_hours = (total_offset_seconds/3600)|int  %}
+  {% set total_offset_days = (total_offset_hours/24)|int  %}
+
+  {% if granularity == "day" %}
+    {% set total_offset = total_offset_days  %}
+    {% set fmt_out = '"%Y-%m-%d"' %}
+  {% elif  granularity == "hour" %}
+    {% set total_offset = total_offset_hours %}
+    {% set fmt_out = '"%Y-%m-%d %H:%M:%S"' %}
+  {% endif %}
+
+  {% set date_list = [] %}
+  {% for i in range(0, total_offset ) -%}
+    {% if granularity == "day" %}
+        {% set delta = modules.datetime.timedelta(days = i) %}
+    {% elif granularity == "hour" %}
+        {% set delta = modules.datetime.timedelta(hours = i) %}
+    {% endif %}
+    {% set this_date = start_date_datetime + delta %}
+    {% set _ = date_list.append(this_date.strftime(fmt_out)) %}
+  {% endfor -%}
+    {{ log("date_list: " ~ date_list,info=true) }}
+  {{ return(date_list) }}
+{% endmacro %}

dbt-bigquery/src/dbt/include/bigquery/macros/materializations/incremental.sql

@@ -69,6 +69,16 @@
 
 {% endmacro %}
 
+
+{% macro set_partitions(strategy, granularity) %}
+  {#-- We override the partitions to force a static insert overwrite on microbatch, significantly more performant --#}
+  {% if strategy == "microbatch" %}
+    {{ return(bq_generate_static_partitions(config, granularity)) }}
+  {% else %}
+    {{ return(config.get('partitions', none)) }}
+  {% endif %}
+{% endmacro %}
+
 {% materialization incremental, adapter='bigquery', supported_languages=['sql', 'python'] -%}
 
   {%- set unique_key = config.get('unique_key') -%}
@@ -84,7 +94,7 @@
 
   {%- set raw_partition_by = config.get('partition_by', none) -%}
   {%- set partition_by = adapter.parse_partition_by(raw_partition_by) -%}
-  {%- set partitions = config.get('partitions', none) -%}
+  {%- set partitions = set_partitions(strategy, partition_by.granularity) -%}
   {%- set cluster_by = config.get('cluster_by', none) -%}
 
   {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}

dbt-bigquery/src/dbt/include/bigquery/macros/materializations/incremental_strategy/microbatch.sql

@@ -26,3 +26,11 @@
 
     {{ return(build_sql) }}
 {% endmacro %}
+
+{% macro bq_generate_static_partitions(config, granularity) %}
+  {% set event_time_start = config.get("__dbt_internal_microbatch_event_time_start") %}
+  {% set event_time_end = config.get("__dbt_internal_microbatch_event_time_end") %}
+  {% set range_of_dates = generate_dates_in_range(event_time_start, event_time_end, granularity) %}
+
+  {{ return(range_of_dates) }}
+{% endmacro %}