Skip to content

Commit

Permalink
Add unload macro example
Browse files Browse the repository at this point in the history
  • Loading branch information
dpguthrie committed Feb 16, 2024
1 parent 82b46c8 commit 6e51674
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 18 deletions.
18 changes: 0 additions & 18 deletions macros/snowflake/dont_do_this.sql

This file was deleted.

39 changes: 39 additions & 0 deletions macros/snowflake/unloading.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{% macro create_unload_external_stage() %}
{% if env_var('DBT_ENVIRONMENT', 'dev') == 'prod' %}
{% set sql %}
CREATE OR REPLACE STAGE dbt_ext_unload_stage
url='s3://your-company-artifacts/snowflake-unload/dbt/'
credentials=(aws_key_id='{{ env_var('DBT_SECRET_AWS_ACCESS_KEY_ID') }}', aws_secret_key='{{ env_var('DBT_SECRET_AWS_SECRET_ACCESS_KEY') }}')
file_format = (type = csv, field_optionally_enclosed_by = '"', compression = gzip);
{% endset %}
{% set _ = run_query(sql) %}
{% endif %}
{% endmacro %}

{% macro unload_to_s3() %}
-- Another example here: https://gist.github.com/jeremyyeo/f07dbe9a7687ffc4976e1488a8e35547

{% if env_var('DBT_ENVIRONMENT', 'dev') == 'prod' %}

{% set sql %}

-- Unload (possibly multiple) data files into timestamped directory.
COPY INTO @dbt_ext_unload_stage/{{ this.name }}/{{ run_started_at.date().isoformat() }}/{{ run_started_at.isoformat() }}/data
FROM {{ this }}
header = true;

-- Add an indicator that the unload has completed.
COPY INTO @dbt_ext_unload_stage/{{ this.name }}/{{ run_started_at.date().isoformat() }}/{{ run_started_at.isoformat() }}/done.csv
FROM (SELECT 1 AS done)
single = true;

{% endset %}

{% if execute %}

{% do run_query(sql) %}

{% endif %}

{% endif %}
{% endmacro %}

0 comments on commit 6e51674

Please sign in to comment.