Skip to content
Data Source Cleanup

Data Source Cleanup #3012

Sign in to view logs

Data Source Cleanup

Data Source Cleanup #3012

Workflow file for this run

.github/workflows/data_source_cleanup.yml at fc6ad13
# Script(s) to clean up any loose data left behind from test runs.
# These can primarily happen if CI is stopped while tests are running
# for big query, etc.
name: Data Source Cleanup
on:
workflow_dispatch:
schedule:
- cron: "0 * * * *" # Run at the start of each hour
jobs:
cleanup-big-query:
runs-on: ubuntu-latest
env:
# google
GE_TEST_GCP_CREDENTIALS: ${{secrets.GE_TEST_GCP_CREDENTIALS}}
GE_TEST_GCP_PROJECT: ${{secrets.GE_TEST_GCP_PROJECT}}
GE_TEST_BIGQUERY_DATASET: ${{secrets.GE_TEST_BIGQUERY_DATASET}}
GOOGLE_APPLICATION_CREDENTIALS: "gcp-credentials.json"
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Create JSON file GCP
run: |
echo "$GE_TEST_GCP_CREDENTIALS" > gcp-credentials.json
- name: Install dependencies
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -m bigquery
pip install -e .
- name: Run BigQuery cleanup script
run: |
python ./scripts/cleanup/cleanup_big_query.py
cleanup-redshift:
runs-on: ubuntu-latest
env:
# aws-redshift
REDSHIFT_USERNAME: ${{secrets.REDSHIFT_USERNAME}}
REDSHIFT_PASSWORD: ${{secrets.REDSHIFT_PASSWORD}}
REDSHIFT_HOST: ${{secrets.REDSHIFT_HOST}}
REDSHIFT_PORT: ${{secrets.REDSHIFT_PORT}}
REDSHIFT_DATABASE: ${{secrets.REDSHIFT_DATABASE}}
REDSHIFT_SSLMODE: ${{secrets.REDSHIFT_SSLMODE}}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Install dependencies
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -m gx-redshift
pip install -e .
- name: Run Redshift cleanup script
run: |
python ./scripts/cleanup/cleanup_redshift.py
cleanup-databricks:
runs-on: ubuntu-latest
env:
# databricks
DATABRICKS_TOKEN: ${{secrets.DATABRICKS_SQL_TOKEN}}
DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
DATABRICKS_HTTP_PATH: ${{secrets.DATABRICKS_HTTP_PATH}}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Install dependencies
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -m databricks
pip install -e .
- name: Run Databricks cleanup script
run: |
python ./scripts/cleanup/cleanup_databricks.py
cleanup-snowflake:
runs-on: ubuntu-latest
env:
# snowflake
SNOWFLAKE_CI_ACCOUNT: ${{secrets.SNOWFLAKE_CI_ACCOUNT}}
SNOWFLAKE_CI_USER_PASSWORD: ${{secrets.SNOWFLAKE_CI_USER_PASSWORD}}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: "pip"
cache-dependency-path: |
reqs/requirements-dev-test.txt
setup.py
- name: Install dependencies
run: |
pip install $(grep -E '^(invoke)' reqs/requirements-dev-contrib.txt)
invoke deps --gx-install -m snowflake
pip install -e .
- name: Run Snowflake cleanup script
run: |
python ./scripts/cleanup/cleanup_snowflake.py