From bc0add86641a8cb44dcb797889698a8aa8510b90 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 08:57:32 -0600 Subject: [PATCH 01/21] wiup: snowflake tests with secrets --- .github/ci.yml | 44 ++++++++++++++++++ supported_adapters.env | 1 + tox.ini | 101 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 100644 .github/ci.yml create mode 100644 supported_adapters.env create mode 100644 tox.ini diff --git a/.github/ci.yml b/.github/ci.yml new file mode 100644 index 00000000..89d4cc05 --- /dev/null +++ b/.github/ci.yml @@ -0,0 +1,44 @@ +# **what?** +# Run tests for dbt-utils against supported adapters + +# **why?** +# To ensure that dbt-utils works as expected with all supported adapters + +# **when?** +# On every PR, and every push to main and when manually triggered + +name: Package Integration Tests + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + + +jobs: + run-tests: + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1 + with: + # no need to pass postgres vars in. We can just use the defaults in the local container + # redshift + REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}" + REDSHIFT_PORT: 5439 + # bigquery + BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} + BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}" + # snowflake + SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} + SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }} + SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }} + SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}" + secrets: + DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} + BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} \ No newline at end of file diff --git a/supported_adapters.env b/supported_adapters.env new file mode 100644 index 00000000..fc4c9ff3 --- /dev/null +++ b/supported_adapters.env @@ -0,0 +1 @@ +SUPPORTED_ADAPTERS=snowflake \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..3e8cb19c --- /dev/null +++ b/tox.ini @@ -0,0 +1,101 @@ +[tox] +skipsdist = True +envlist = lint_all, testenv + +[testenv] +passenv = + # postgres env vars + POSTGRES_HOST + POSTGRES_USER + DBT_ENV_SECRET_POSTGRES_PASS + POSTGRES_PORT + POSTGRES_DATABASE + POSTGRES_SCHEMA + # snowflake env vars + SNOWFLAKE_ACCOUNT + SNOWFLAKE_USER + DBT_ENV_SECRET_SNOWFLAKE_PASS + SNOWFLAKE_ROLE + SNOWFLAKE_DATABASE + SNOWFLAKE_WAREHOUSE + SNOWFLAKE_SCHEMA + # redshift + REDSHIFT_HOST + REDSHIFT_USER + DBT_ENV_SECRET_REDSHIFT_PASS + REDSHIFT_DATABASE + REDSHIFT_SCHEMA + REDSHIFT_PORT + # bigquery + BIGQUERY_PROJECT + BIGQUERY_KEYFILE_JSON + BIGQUERY_SCHEMA + +# Snowflake integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_snowflake_1] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt deps --target snowflake + dbt build -x --target snowflake --full-refresh + + # test with the second project + cd ../integration_tests_2 + dbt deps --target snowflake + dbt seed --full-refresh --target snowflake + dbt run -x --target snowflake --full-refresh + + +# Postgres integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_postgres] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt deps --target postgres + dbt build -x --target postgres --full-refresh + + # test with the second project + cd ../integration_tests_2 + dbt deps --target postgres + dbt seed --full-refresh --target postgres + dbt run -x --target postgres --full-refresh + +# BigQuery integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_bigquery] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt deps --target bigquery + dbt build -x --target bigquery --full-refresh + + # test with the second project + cd ../integration_tests_2 + dbt deps --target bigquery + dbt seed --full-refresh --target bigquery + dbt run -x --target bigquery --full-refresh + +# redshift integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_redshift] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt deps --target redshift + dbt build -x --target redshift --full-refresh + + # test with the second project + cd ../integration_tests_2 + dbt deps --target redshift + dbt seed --full-refresh --target redshift + dbt run -x --target redshift --full-refresh \ No newline at end of file From 34276d2826ff4b9147ef6bf423ec3f3803ce1c6b Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 08:59:27 -0600 Subject: [PATCH 02/21] rename workflow --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3e8cb19c..27c7610b 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ passenv = # Snowflake integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment -[testenv:dbt_integration_snowflake_1] +[testenv:dbt_integration_snowflake] changedir = integration_tests allowlist_externals = dbt From a32ee4e7288d8bb38912a8eddcefd6cb31e4a345 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 09:00:45 -0600 Subject: [PATCH 03/21] add profiles to root of test dir --- .gitignore | 2 +- integration_tests/profiles.yml | 84 ++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 integration_tests/profiles.yml diff --git a/.gitignore b/.gitignore index 480a4422..4228318b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ dbt_packages/ integration_tests/state/ site/ env/ -profiles.yml +# profiles.yml package-lock.yml # IDE diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml new file mode 100644 index 00000000..dff42772 --- /dev/null +++ b/integration_tests/profiles.yml @@ -0,0 +1,84 @@ + +# HEY! This file is used in the integration tests with CircleCI. +# You should __NEVER__ check credentials into version control. Thanks for reading :) + +config: + send_anonymous_usage_stats: False + use_colors: True + +integration_tests: + target: postgres + outputs: + postgres: + type: postgres + host: "{{ env_var('POSTGRES_TEST_HOST') }}" + user: "{{ env_var('POSTGRES_TEST_USER') }}" + pass: "{{ env_var('POSTGRES_TEST_PASS') }}" + port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" + schema: dbt_project_evaluator_integration_tests_postgres + threads: 5 + + redshift: + type: redshift + host: "{{ env_var('REDSHIFT_TEST_HOST') }}" + user: "{{ env_var('REDSHIFT_TEST_USER') }}" + pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" + dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" + port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" + schema: dbt_project_evaluator_integration_tests_redshift + threads: 5 + + bigquery: + type: bigquery + method: service-account + keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" + project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" + schema: dbt_project_evaluator_integration_tests_bigquery + threads: 10 + + snowflake: + type: snowflake + account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" + password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" + role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" + schema: dbt_project_evaluator_integration_tests_snowflake + threads: 10 + + databricks: + type: databricks + schema: dbt_project_evaluator_integration_tests_databricks + host: "{{ env_var('DATABRICKS_TEST_HOST') }}" + http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" + token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}" + threads: 10 + + duckdb: + type: duckdb + path: ./duck.db + + trino: + type: trino + host: "{{ env_var('TRINO_TEST_HOST') }}" + port: "{{ env_var('TRINO_TEST_PORT') | as_number }}" + method: ldap + user: "{{ env_var('TRINO_TEST_USER') }}" + password: "{{ env_var('TRINO_TEST_PASS') }}" + catalog: "{{ env_var('TRINO_TEST_CATALOG_NAME') }}" + schema: dbt_project_evaluator_integration_tests_trino + threads: 5 + session_properties: + query_max_stage_count: 275 + + clickhouse: + type: clickhouse + host: "{{ env_var('CLICKHOUSE_TEST_HOST') }}" + port: "{{ env_var('CLICKHOUSE_TEST_PORT') | as_number }}" + user: "{{ env_var('CLICKHOUSE_TEST_USER') }}" + password: "{{ env_var('CLICKHOUSE_TEST_PASS') }}" + dbname: "{{ env_var('CLICKHOUSE_TEST_DBNAME') }}" + schema: dbt_project_evaluator_integration_tests_clickhouse + threads: 5 From 337a5d3cc39c1986bbe741bbc9136de8134e42ba Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 09:01:20 -0600 Subject: [PATCH 04/21] workflow moved to correct dir --- .github/{ => workflows}/ci.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/ci.yml (100%) diff --git a/.github/ci.yml b/.github/workflows/ci.yml similarity index 100% rename from .github/ci.yml rename to .github/workflows/ci.yml From 08f026362b68f44829b559d25e6d3066a25cd1fe Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 10:19:23 -0600 Subject: [PATCH 05/21] update with vars --- .github/workflows/ci.yml | 8 +++--- .gitignore | 3 ++- integration_tests/profiles.yml | 47 +++++++++++++++++----------------- tox.ini | 38 ++++++++++++++++++++++++++- 4 files changed, 67 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89d4cc05..cb299686 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,10 +32,10 @@ jobs: BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}" # snowflake - SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} - SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }} - SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }} - SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }} + SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} + SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}" secrets: DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} diff --git a/.gitignore b/.gitignore index 4228318b..4f214080 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,8 @@ venv/ # Environment variables -.env +# .env +test.env # DBT artifacts target/ diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index dff42772..50e2a231 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -11,46 +11,47 @@ integration_tests: outputs: postgres: type: postgres - host: "{{ env_var('POSTGRES_TEST_HOST') }}" - user: "{{ env_var('POSTGRES_TEST_USER') }}" - pass: "{{ env_var('POSTGRES_TEST_PASS') }}" - port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" - dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" - schema: dbt_project_evaluator_integration_tests_postgres + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_test') }}" threads: 5 redshift: type: redshift - host: "{{ env_var('REDSHIFT_TEST_HOST') }}" - user: "{{ env_var('REDSHIFT_TEST_USER') }}" - pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" - dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" - port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" - schema: dbt_project_evaluator_integration_tests_redshift + host: "{{ env_var('REDSHIFT_HOST') }}" + user: "{{ env_var('REDSHIFT_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" + dbname: "{{ env_var('REDSHIFT_DATABASE') }}" + port: "{{ env_var('REDSHIFT_PORT') | as_number }}" + schema: "{{ env_var('REDSHIFT_SCHEMA') }}" threads: 5 bigquery: type: bigquery method: service-account - keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" - project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" - schema: dbt_project_evaluator_integration_tests_bigquery + keyfile_json: + "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" + project: "{{ env_var('BIGQUERY_PROJECT') }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" threads: 10 snowflake: type: snowflake - account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" - user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" - password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" - role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" - database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" - warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" - schema: dbt_project_evaluator_integration_tests_snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" + role: "{{ env_var('SNOWFLAKE_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" threads: 10 databricks: type: databricks - schema: dbt_project_evaluator_integration_tests_databricks + schema: "{{ env_var('DATABRICKS_SCHEMA', 'dbt_project_evaluator_integration_tests_databricks') }}" host: "{{ env_var('DATABRICKS_TEST_HOST') }}" http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}" diff --git a/tox.ini b/tox.ini index 27c7610b..90738ef7 100644 --- a/tox.ini +++ b/tox.ini @@ -37,6 +37,7 @@ passenv = changedir = integration_tests allowlist_externals = dbt + cd skip_install = true commands = dbt deps --target snowflake @@ -55,6 +56,7 @@ commands = changedir = integration_tests allowlist_externals = dbt + cd skip_install = true commands = dbt deps --target postgres @@ -72,6 +74,7 @@ commands = changedir = integration_tests allowlist_externals = dbt + cd skip_install = true commands = dbt deps --target bigquery @@ -89,6 +92,7 @@ commands = changedir = integration_tests allowlist_externals = dbt + cd skip_install = true commands = dbt deps --target redshift @@ -98,4 +102,36 @@ commands = cd ../integration_tests_2 dbt deps --target redshift dbt seed --full-refresh --target redshift - dbt run -x --target redshift --full-refresh \ No newline at end of file + dbt run -x --target redshift --full-refresh + +[testenv:dbt_integration_duckdb] +changedir = integration_tests +allowlist_externals = + dbt + cd +skip_install = true +commands = + dbt deps --target duckdb + dbt build -x --target duckdb --full-refresh + + # test with the second project + cd ../integration_tests_2 + dbt deps --target duckdb + dbt seed --full-refresh --target duckdb + dbt run -x --target duckdb --full-refresh + +[testenv:dbt_integration_databricks] +changedir = integration_tests +allowlist_externals = + dbt + cd +skip_install = true +commands = + dbt deps --target duckdb + dbt build -x --target duckdb --full-refresh + + # test with the second project + cd ../integration_tests_2 + dbt deps --target duckdb + dbt seed --full-refresh --target duckdb + dbt run -x --target duckdb --full-refresh \ No newline at end of file From a72500e494368afe2e8fc39bfcd6de8fd9d162ec Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 10:40:35 -0600 Subject: [PATCH 06/21] account not specified in workflow --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb299686..d530688d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}" secrets: DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} From fa2d39dfb5d02c073a65415213abb2d504381b89 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 10:43:11 -0600 Subject: [PATCH 07/21] remove var --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d530688d..cb299686 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,6 @@ jobs: SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} - SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_ACCOUNT }} SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}" secrets: DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} From c303c4027a68e3443b0a23041ba5de036148de7b Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 13 Nov 2024 11:30:11 -0600 Subject: [PATCH 08/21] test script instead of inline tox --- run_tox_test.sh | 12 ++++++++++++ tox.ini | 13 ++----------- 2 files changed, 14 insertions(+), 11 deletions(-) create mode 100755 run_tox_test.sh diff --git a/run_tox_test.sh b/run_tox_test.sh new file mode 100755 index 00000000..4ecfe0c4 --- /dev/null +++ b/run_tox_test.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# test with the first project +cd integration_tests +dbt deps --target $1 || exit 1 +dbt build -x --target $1 --full-refresh || exit 1 + +# test with the second project +cd ../integration_tests_2 +dbt deps --target $1 || exit 1 +dbt seed --full-refresh --target $1 || exit 1 +dbt run -x --target $1 --full-refresh || exit 1 diff --git a/tox.ini b/tox.ini index 90738ef7..c4424d97 100644 --- a/tox.ini +++ b/tox.ini @@ -34,20 +34,11 @@ passenv = # Snowflake integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment [testenv:dbt_integration_snowflake] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target snowflake - dbt build -x --target snowflake --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target snowflake - dbt seed --full-refresh --target snowflake - dbt run -x --target snowflake --full-refresh + bash ./run_tox_tests.sh snowflake # Postgres integration tests for centralized dbt testing From 77d26a3680883a3351beedd344a5650f5a1d7dc4 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 13 Nov 2024 11:37:31 -0600 Subject: [PATCH 09/21] attempt to fix circleCI --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 19147aca..30256736 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,11 +13,11 @@ jobs: - run: name: "Run Tests - Postgres" environment: - POSTGRES_TEST_HOST: localhost - POSTGRES_TEST_USER: root - POSTGRES_TEST_PASS: '' - POSTGRES_TEST_PORT: 5432 - POSTGRES_TEST_DBNAME: circle_test + POSTGRES_HOST: localhost + POSTGRES_USER: root + POSTGRES_PASS: '' + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: circle_test command: ./run_test.sh postgres - store_artifacts: path: ./integration_tests/logs From f9e80403df39163d7c26c4c1b440bcf5ddb75406 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 13 Nov 2024 11:37:44 -0600 Subject: [PATCH 10/21] rename files, set all commands --- run_tox_test.sh => run_tox_tests.sh | 0 tox.ini | 67 ++++++----------------------- 2 files changed, 14 insertions(+), 53 deletions(-) rename run_tox_test.sh => run_tox_tests.sh (100%) diff --git a/run_tox_test.sh b/run_tox_tests.sh similarity index 100% rename from run_tox_test.sh rename to run_tox_tests.sh diff --git a/tox.ini b/tox.ini index c4424d97..a6b29275 100644 --- a/tox.ini +++ b/tox.ini @@ -40,42 +40,23 @@ skip_install = true commands = bash ./run_tox_tests.sh snowflake - # Postgres integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment [testenv:dbt_integration_postgres] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target postgres - dbt build -x --target postgres --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target postgres - dbt seed --full-refresh --target postgres - dbt run -x --target postgres --full-refresh + bash ./run_tox_tests.sh postgres # BigQuery integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment [testenv:dbt_integration_bigquery] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target bigquery - dbt build -x --target bigquery --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target bigquery - dbt seed --full-refresh --target bigquery - dbt run -x --target bigquery --full-refresh + bash ./run_tox_tests.sh bigquery # redshift integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment @@ -86,43 +67,23 @@ allowlist_externals = cd skip_install = true commands = - dbt deps --target redshift - dbt build -x --target redshift --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target redshift - dbt seed --full-refresh --target redshift - dbt run -x --target redshift --full-refresh +allowlist_externals = + bash +skip_install = true +commands = + bash ./run_tox_tests.sh redshift +# note that duckdb is not a supported dbt target for dbt Cloud testing [testenv:dbt_integration_duckdb] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target duckdb - dbt build -x --target duckdb --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target duckdb - dbt seed --full-refresh --target duckdb - dbt run -x --target duckdb --full-refresh + bash ./run_tox_tests.sh duckdb [testenv:dbt_integration_databricks] -changedir = integration_tests allowlist_externals = - dbt - cd + bash skip_install = true commands = - dbt deps --target duckdb - dbt build -x --target duckdb --full-refresh - - # test with the second project - cd ../integration_tests_2 - dbt deps --target duckdb - dbt seed --full-refresh --target duckdb - dbt run -x --target duckdb --full-refresh \ No newline at end of file + bash ./run_tox_tests.sh databricks \ No newline at end of file From 450eb1cad010fa5092b6076001825cd14ca40e1e Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 13 Nov 2024 11:40:26 -0600 Subject: [PATCH 11/21] oops --- .circleci/config.yml | 2 +- tox.ini | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 30256736..4c920012 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: environment: POSTGRES_HOST: localhost POSTGRES_USER: root - POSTGRES_PASS: '' + DBT_ENV_SECRET_POSTGRES_PASS: '' POSTGRES_PORT: 5432 POSTGRES_DATABASE: circle_test command: ./run_test.sh postgres diff --git a/tox.ini b/tox.ini index a6b29275..24e1d645 100644 --- a/tox.ini +++ b/tox.ini @@ -61,12 +61,6 @@ commands = # redshift integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment [testenv:dbt_integration_redshift] -changedir = integration_tests -allowlist_externals = - dbt - cd -skip_install = true -commands = allowlist_externals = bash skip_install = true From 8f1318e2f1bcd06b3ec7f178e97ae1e8615e7541 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 13 Nov 2024 11:48:56 -0600 Subject: [PATCH 12/21] add profiles to 2nd test folder --- integration_tests/profiles.yml | 10 ++-- .../profiles.yml | 49 ++++++++++--------- run_test.sh | 6 --- run_tox_tests.sh | 4 +- 4 files changed, 32 insertions(+), 37 deletions(-) rename integration_tests/ci/sample.profiles.yml => integration_tests_2/profiles.yml (50%) diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 50e2a231..f4b73176 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -1,5 +1,5 @@ -# HEY! This file is used in the integration tests with CircleCI. +# HEY! This file is used in the integration tests with CI. # You should __NEVER__ check credentials into version control. Thanks for reading :) config: @@ -16,7 +16,7 @@ integration_tests: pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" port: "{{ env_var('POSTGRES_PORT') | as_number }}" dbname: "{{ env_var('POSTGRES_DATABASE') }}" - schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_test') }}" + schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_integration_tests_postgres') }}" threads: 5 redshift: @@ -26,7 +26,7 @@ integration_tests: pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" dbname: "{{ env_var('REDSHIFT_DATABASE') }}" port: "{{ env_var('REDSHIFT_PORT') | as_number }}" - schema: "{{ env_var('REDSHIFT_SCHEMA') }}" + schema: "{{ env_var('REDSHIFT_SCHEMA', 'dbt_project_evaluator_integration_tests_redshift') }}" threads: 5 bigquery: @@ -35,7 +35,7 @@ integration_tests: keyfile_json: "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" project: "{{ env_var('BIGQUERY_PROJECT') }}" - dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}" threads: 10 snowflake: @@ -46,7 +46,7 @@ integration_tests: role: "{{ env_var('SNOWFLAKE_ROLE') }}" database: "{{ env_var('SNOWFLAKE_DATABASE') }}" warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" - schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA', 'dbt_project_evaluator_integration_tests_snowflake') }}" threads: 10 databricks: diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests_2/profiles.yml similarity index 50% rename from integration_tests/ci/sample.profiles.yml rename to integration_tests_2/profiles.yml index dff42772..f4b73176 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests_2/profiles.yml @@ -1,5 +1,5 @@ -# HEY! This file is used in the integration tests with CircleCI. +# HEY! This file is used in the integration tests with CI. # You should __NEVER__ check credentials into version control. Thanks for reading :) config: @@ -11,46 +11,47 @@ integration_tests: outputs: postgres: type: postgres - host: "{{ env_var('POSTGRES_TEST_HOST') }}" - user: "{{ env_var('POSTGRES_TEST_USER') }}" - pass: "{{ env_var('POSTGRES_TEST_PASS') }}" - port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" - dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" - schema: dbt_project_evaluator_integration_tests_postgres + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA', 'dbt_project_evaluator_integration_tests_postgres') }}" threads: 5 redshift: type: redshift - host: "{{ env_var('REDSHIFT_TEST_HOST') }}" - user: "{{ env_var('REDSHIFT_TEST_USER') }}" - pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" - dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" - port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" - schema: dbt_project_evaluator_integration_tests_redshift + host: "{{ env_var('REDSHIFT_HOST') }}" + user: "{{ env_var('REDSHIFT_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" + dbname: "{{ env_var('REDSHIFT_DATABASE') }}" + port: "{{ env_var('REDSHIFT_PORT') | as_number }}" + schema: "{{ env_var('REDSHIFT_SCHEMA', 'dbt_project_evaluator_integration_tests_redshift') }}" threads: 5 bigquery: type: bigquery method: service-account - keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" - project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" - schema: dbt_project_evaluator_integration_tests_bigquery + keyfile_json: + "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" + project: "{{ env_var('BIGQUERY_PROJECT') }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}" threads: 10 snowflake: type: snowflake - account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" - user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" - password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" - role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" - database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" - warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" - schema: dbt_project_evaluator_integration_tests_snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" + role: "{{ env_var('SNOWFLAKE_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA', 'dbt_project_evaluator_integration_tests_snowflake') }}" threads: 10 databricks: type: databricks - schema: dbt_project_evaluator_integration_tests_databricks + schema: "{{ env_var('DATABRICKS_SCHEMA', 'dbt_project_evaluator_integration_tests_databricks') }}" host: "{{ env_var('DATABRICKS_TEST_HOST') }}" http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}" diff --git a/run_test.sh b/run_test.sh index 07b39521..c7e9ed4f 100755 --- a/run_test.sh +++ b/run_test.sh @@ -13,12 +13,6 @@ fi . $VENV cd integration_tests - -if [[ ! -e ~/.dbt/profiles.yml ]]; then - mkdir -p ~/.dbt - cp ci/sample.profiles.yml ~/.dbt/profiles.yml -fi - dbt deps --target $1 || exit 1 dbt build -x --target $1 --full-refresh || exit 1 diff --git a/run_tox_tests.sh b/run_tox_tests.sh index 4ecfe0c4..9b80e093 100755 --- a/run_tox_tests.sh +++ b/run_tox_tests.sh @@ -1,11 +1,11 @@ #!/bin/bash -# test with the first project +echo "Running tests for the first project" cd integration_tests dbt deps --target $1 || exit 1 dbt build -x --target $1 --full-refresh || exit 1 -# test with the second project +echo "Running tests for the second project" cd ../integration_tests_2 dbt deps --target $1 || exit 1 dbt seed --full-refresh --target $1 || exit 1 From e0d0d10d7e0f687897f19f54b7f403a4f4ca082c Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 13 Nov 2024 12:05:27 -0600 Subject: [PATCH 13/21] add ability to run non-cloud tests via tox --- .github/workflows/local_only.yml | 82 ++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 .github/workflows/local_only.yml diff --git a/.github/workflows/local_only.yml b/.github/workflows/local_only.yml new file mode 100644 index 00000000..892ea96c --- /dev/null +++ b/.github/workflows/local_only.yml @@ -0,0 +1,82 @@ +# **what?** +# Run tests for packages not supported for cloud testing +# +# **why?** +# To ensure that packages works as expected with all supported adapters + +# **when?** +# On push, PR or manually called + + +name: Package Integration Tests - Local Only + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +env: + PYTHON_VERSION: "3.11" + POSTGRES_HOST: "localhost" + POSTGRES_USER: "root" + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: "postgres_test" + DBT_ENV_SECRET_POSTGRES_PASS: "password" # this isn't actually a secret since it only runs on the runner + +jobs: + run-tests: + runs-on: ubuntu-latest + services: + postgres: + image: postgres + env: + POSTGRES_USER: ${{ env.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }} + POSTGRES_DB: ${{ env.POSTGRES_DATABASE }} + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + strategy: + fail-fast: false + matrix: + # these adapters are tested in this repo but are not tested as part of the dbt Cloud images. + adapter: [duckdb, postgres] + + steps: + - name: "Checkout ${{ github.event.repository }} " + uses: actions/checkout@v4 + + - name: "Set up Python ${{ env.PYTHON_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: "Install ${{ matrix.adapter }}" + run: | + python -m pip install --upgrade pip + pip install dbt-${{ matrix.adapter }} + + - name: "Install tox" + run: | + python -m pip install --upgrade pip + pip install tox + + - name: "Run integration tests with tox on ${{ matrix.adapter }}" + run: | + tox -e dbt_integration_${{ matrix.adapter }} + env: + # postgres + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} + POSTGRES_USER: ${{ env.POSTGRES_USER }} + DBT_ENV_SECRET_POSTGRES_PASS: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }} + POSTGRES_PORT: ${{ env.POSTGRES_PORT }} + POSTGRES_DATABASE: ${{ env.POSTGRES_DATABASE }} + POSTGRES_SCHEMA: "integration_tests_postgres_${{ github.run_number }}" + # duckdb - needs no vars From da47e02ce1af897ec95c9ec6a04bb8221b2284a9 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 13 Nov 2024 12:07:25 -0600 Subject: [PATCH 14/21] add calrifying comment --- .github/workflows/local_only.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/local_only.yml b/.github/workflows/local_only.yml index 892ea96c..2449cb3d 100644 --- a/.github/workflows/local_only.yml +++ b/.github/workflows/local_only.yml @@ -47,6 +47,7 @@ jobs: fail-fast: false matrix: # these adapters are tested in this repo but are not tested as part of the dbt Cloud images. + # This list should include anything not listed in supported_adapters.env adapter: [duckdb, postgres] steps: From f55be3924835d7375d81d07c7a4784b04c8b9f25 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 15:21:54 -0600 Subject: [PATCH 15/21] add bq --- .github/workflows/ci.yml | 4 ++-- supported_adapters.env | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb299686..a36ade4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,11 +26,11 @@ jobs: REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} - REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}" + REDSHIFT_SCHEMA: "dpe_integration_tests_redshift_${{ github.run_number }}" REDSHIFT_PORT: 5439 # bigquery BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} - BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}" + BIGQUERY_SCHEMA: "dpe_integration_tests_bigquery_${{ github.run_number }}" # snowflake SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} diff --git a/supported_adapters.env b/supported_adapters.env index fc4c9ff3..c1a4e071 100644 --- a/supported_adapters.env +++ b/supported_adapters.env @@ -1 +1 @@ -SUPPORTED_ADAPTERS=snowflake \ No newline at end of file +SUPPORTED_ADAPTERS=snowflake,bigquery \ No newline at end of file From a242a53b4a2317b93ade6dab976935e77296e41f Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 15:35:04 -0600 Subject: [PATCH 16/21] whitespace in yml --- integration_tests/profiles.yml | 3 +-- integration_tests_2/profiles.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index f4b73176..a4b4eabf 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -32,8 +32,7 @@ integration_tests: bigquery: type: bigquery method: service-account - keyfile_json: - "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" + keyfile_json: "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" project: "{{ env_var('BIGQUERY_PROJECT') }}" dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}" threads: 10 diff --git a/integration_tests_2/profiles.yml b/integration_tests_2/profiles.yml index f4b73176..a4b4eabf 100644 --- a/integration_tests_2/profiles.yml +++ b/integration_tests_2/profiles.yml @@ -32,8 +32,7 @@ integration_tests: bigquery: type: bigquery method: service-account - keyfile_json: - "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" + keyfile_json: "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" project: "{{ env_var('BIGQUERY_PROJECT') }}" dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}" threads: 10 From 42e7e17a2c4c884258224f18cd3c43908eab1fc9 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 15:39:36 -0600 Subject: [PATCH 17/21] add redshift just for fun --- supported_adapters.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supported_adapters.env b/supported_adapters.env index c1a4e071..28dfe9f6 100644 --- a/supported_adapters.env +++ b/supported_adapters.env @@ -1 +1 @@ -SUPPORTED_ADAPTERS=snowflake,bigquery \ No newline at end of file +SUPPORTED_ADAPTERS=snowflake,bigquery,redshift \ No newline at end of file From 4b42bb9fde1b4b2013780f20fbbd682ef0d782aa Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 15:51:45 -0600 Subject: [PATCH 18/21] bq profiles --- integration_tests/profiles.yml | 2 +- integration_tests_2/profiles.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index a4b4eabf..31a43ee2 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -31,7 +31,7 @@ integration_tests: bigquery: type: bigquery - method: service-account + method: service-account-json keyfile_json: "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" project: "{{ env_var('BIGQUERY_PROJECT') }}" dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}" diff --git a/integration_tests_2/profiles.yml b/integration_tests_2/profiles.yml index a4b4eabf..31a43ee2 100644 --- a/integration_tests_2/profiles.yml +++ b/integration_tests_2/profiles.yml @@ -31,7 +31,7 @@ integration_tests: bigquery: type: bigquery - method: service-account + method: service-account-json keyfile_json: "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" project: "{{ env_var('BIGQUERY_PROJECT') }}" dataset: "{{ env_var('BIGQUERY_SCHEMA', 'dbt_project_evaluator_integration_tests_bigquery') }}" From 045ff180d33616fdb5833b59b5b09f5c6d400aac Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 16:10:49 -0600 Subject: [PATCH 19/21] dbx --- .github/workflows/ci.yml | 7 ++++++- integration_tests/profiles.yml | 6 +++--- supported_adapters.env | 2 +- tox.ini | 5 +++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a36ade4c..087dac4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,13 @@ jobs: SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}" + # databricks + DATABRICKS_SCHEMA: "integration_tests_databricks_${{ github.run_number }}" + DATABRICKS_HOST: ${{ vars.DATABRICKS_HOST }} + DATABRICKS_HTTP_PATH: ${{ vars.DATABRICKS_HTTP_PATH }} secrets: DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} \ No newline at end of file + DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} + DBT_SECRET_ENV_DATABRICKS_TOKEN: ${{ secrets.DBT_SECRET_ENV_DATABRICKS_TOKEN }} \ No newline at end of file diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 31a43ee2..8270990d 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -51,9 +51,9 @@ integration_tests: databricks: type: databricks schema: "{{ env_var('DATABRICKS_SCHEMA', 'dbt_project_evaluator_integration_tests_databricks') }}" - host: "{{ env_var('DATABRICKS_TEST_HOST') }}" - http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" - token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}" + host: "{{ env_var('DATABRICKS_HOST') }}" + http_path: "{{ env_var('DATABRICKS_HTTP_PATH') }}" + token: "{{ env_var('DBT_SECRET_ENV_DATABRICKS_TOKEN') }}" threads: 10 duckdb: diff --git a/supported_adapters.env b/supported_adapters.env index 28dfe9f6..83de7964 100644 --- a/supported_adapters.env +++ b/supported_adapters.env @@ -1 +1 @@ -SUPPORTED_ADAPTERS=snowflake,bigquery,redshift \ No newline at end of file +SUPPORTED_ADAPTERS=snowflake,bigquery,redshift,databricks \ No newline at end of file diff --git a/tox.ini b/tox.ini index 24e1d645..aa2e3f24 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,11 @@ passenv = BIGQUERY_PROJECT BIGQUERY_KEYFILE_JSON BIGQUERY_SCHEMA + # databricks + DATABRICKS_SCHEMA + DATABRICKS_HOST + DATABRICKS_HTTP_PATH + DBT_SECRET_ENV_DATABRICKS_TOKEN # Snowflake integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment From 464ca23576ef019c2ba68c39cac8ad3773779af8 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 13 Nov 2024 16:19:19 -0600 Subject: [PATCH 20/21] other profile --- integration_tests_2/profiles.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests_2/profiles.yml b/integration_tests_2/profiles.yml index 31a43ee2..8270990d 100644 --- a/integration_tests_2/profiles.yml +++ b/integration_tests_2/profiles.yml @@ -51,9 +51,9 @@ integration_tests: databricks: type: databricks schema: "{{ env_var('DATABRICKS_SCHEMA', 'dbt_project_evaluator_integration_tests_databricks') }}" - host: "{{ env_var('DATABRICKS_TEST_HOST') }}" - http_path: "{{ env_var('DATABRICKS_TEST_HTTP_PATH') }}" - token: "{{ env_var('DATABRICKS_TEST_ACCESS_TOKEN') }}" + host: "{{ env_var('DATABRICKS_HOST') }}" + http_path: "{{ env_var('DATABRICKS_HTTP_PATH') }}" + token: "{{ env_var('DBT_SECRET_ENV_DATABRICKS_TOKEN') }}" threads: 10 duckdb: From 3ebdb6d12fbf50eeef8dd0f93ff1152fa5ace6e3 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 14 Nov 2024 09:47:09 -0600 Subject: [PATCH 21/21] the ole switcheroo --- .github/workflows/ci.yml | 2 +- integration_tests/profiles.yml | 2 +- integration_tests_2/profiles.yml | 2 +- tox.ini | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 087dac4d..b97dafb0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,4 +46,4 @@ jobs: BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} - DBT_SECRET_ENV_DATABRICKS_TOKEN: ${{ secrets.DBT_SECRET_ENV_DATABRICKS_TOKEN }} \ No newline at end of file + DBT_ENV_SECRET_DATABRICKS_TOKEN: ${{ secrets.DBT_ENV_SECRET_DATABRICKS_TOKEN }} \ No newline at end of file diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 8270990d..75ef5c15 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -53,7 +53,7 @@ integration_tests: schema: "{{ env_var('DATABRICKS_SCHEMA', 'dbt_project_evaluator_integration_tests_databricks') }}" host: "{{ env_var('DATABRICKS_HOST') }}" http_path: "{{ env_var('DATABRICKS_HTTP_PATH') }}" - token: "{{ env_var('DBT_SECRET_ENV_DATABRICKS_TOKEN') }}" + token: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_TOKEN') }}" threads: 10 duckdb: diff --git a/integration_tests_2/profiles.yml b/integration_tests_2/profiles.yml index 8270990d..75ef5c15 100644 --- a/integration_tests_2/profiles.yml +++ b/integration_tests_2/profiles.yml @@ -53,7 +53,7 @@ integration_tests: schema: "{{ env_var('DATABRICKS_SCHEMA', 'dbt_project_evaluator_integration_tests_databricks') }}" host: "{{ env_var('DATABRICKS_HOST') }}" http_path: "{{ env_var('DATABRICKS_HTTP_PATH') }}" - token: "{{ env_var('DBT_SECRET_ENV_DATABRICKS_TOKEN') }}" + token: "{{ env_var('DBT_ENV_SECRET_DATABRICKS_TOKEN') }}" threads: 10 duckdb: diff --git a/tox.ini b/tox.ini index aa2e3f24..03e891b0 100644 --- a/tox.ini +++ b/tox.ini @@ -34,7 +34,7 @@ passenv = DATABRICKS_SCHEMA DATABRICKS_HOST DATABRICKS_HTTP_PATH - DBT_SECRET_ENV_DATABRICKS_TOKEN + DBT_ENV_SECRET_DATABRICKS_TOKEN # Snowflake integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment