feat(reporting): migrate meltano to snowflake from bigquery

meltano/.sqlfluff

@@ -1,6 +1,6 @@
 [sqlfluff]
 templater = dbt
-dialect = bigquery
+dialect = snowflake
 exclude_rules = structure.using,references.qualification,references.from
 runaway_limit = 10
 max_line_length = 100
@@ -35,5 +35,5 @@ aliasing = explicit
 
 [sqlfluff:templater:dbt]
 project_dir = transform
-profiles_dir = transform/profiles/bigquery/
+profiles_dir = transform/profiles/snowflake/
 profile = meltano

meltano/meltano.yml

@@ -79,6 +79,19 @@ plugins:
     config:
       project: lana-dev-440721
       generate_view: true
+  - name: target-snowflake
+    variant: meltanolabs
+    pip_url: meltanolabs-target-snowflake
+    config:
+      account: cdagppu-sy37948
+      database: MELTANO_DATABASE
+      user: MELTANO_USER
+      role: MELTANO_ROLE
+      warehouse: MELTANO_WAREHOUSE
+      schema: RAW_SCHEMA
+      default_target_schema: RAW_SCHEMA
+      flattening_enabled: true
+      flattening_max_depth: 10
   transformers:
   - name: dbt-bigquery
     variant: dbt-labs
@@ -89,20 +102,26 @@ plugins:
   utilities:
   - name: sqlfluff
     variant: sqlfluff
-    pip_url: sqlfluff sqlfluff-templater-dbt dbt-core dbt-bigquery
-    settings:
-    - name: dataset
-      env: DBT_BIGQUERY_DATASET
-    - name: keyfile
-      env: DBT_BIGQUERY_KEYFILE
-    - name: auth_method
-      env: DBT_BIGQUERY_AUTH_METHOD
-      value: service-account
-    - name: project
-      env: DBT_BIGQUERY_PROJECT
-      value: lana-dev-440721
+    pip_url: sqlfluff sqlfluff-templater-dbt dbt-core dbt-snowflake
+    config:
+      account: cdagppu-sy37948
+      database: MELTANO_DATABASE
+      role: MELTANO_ROLE
+      schema: DBT_SCHEMA
+      user: MELTANO_USER
+      warehouse: MELTANO_WAREHOUSE
   - name: airflow
     variant: apache
     pip_url: git+https://github.com/meltano/airflow-ext.git@main apache-airflow==2.8.1
-      --constraint
+      --constraint 
       https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-no-providers-${MELTANO__PYTHON_VERSION}.txt
+  - name: dbt-snowflake
+    variant: dbt-labs
+    pip_url: dbt-core dbt-snowflake meltano-dbt-ext~=0.3.0
+    config:
+      account: cdagppu-sy37948
+      database: MELTANO_DATABASE
+      role: MELTANO_ROLE
+      schema: DBT_SCHEMA
+      user: MELTANO_USER
+      warehouse: MELTANO_WAREHOUSE

meltano/plugins/loaders/target-snowflake--meltanolabs.lock

@@ -0,0 +1,205 @@
+{
+  "plugin_type": "loaders",
+  "name": "target-snowflake",
+  "namespace": "target_snowflake",
+  "variant": "meltanolabs",
+  "label": "Snowflake",
+  "docs": "https://hub.meltano.com/loaders/target-snowflake--meltanolabs",
+  "repo": "https://github.com/MeltanoLabs/target-snowflake",
+  "pip_url": "meltanolabs-target-snowflake",
+  "executable": "target-snowflake",
+  "description": "Snowflake database loader",
+  "logo_url": "https://hub.meltano.com/assets/logos/loaders/snowflake.png",
+  "capabilities": [
+    "about",
+    "activate-version",
+    "hard-delete",
+    "schema-flattening",
+    "stream-maps"
+  ],
+  "settings_group_validation": [
+    [
+      "account",
+      "database",
+      "password",
+      "user"
+    ]
+  ],
+  "settings": [
+    {
+      "name": "account",
+      "kind": "string",
+      "label": "Account",
+      "description": "Your account identifier. See [Account Identifiers](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html)."
+    },
+    {
+      "name": "add_record_metadata",
+      "kind": "boolean",
+      "value": true,
+      "label": "Add Record Metadata",
+      "description": "Whether to add metadata columns."
+    },
+    {
+      "name": "batch_size_rows",
+      "kind": "integer",
+      "label": "Batch Size Rows",
+      "description": "Maximum number of rows in each batch."
+    },
+    {
+      "name": "clean_up_batch_files",
+      "kind": "boolean",
+      "value": true,
+      "label": "Clean Up Batch Files",
+      "description": "Whether to remove batch files after processing."
+    },
+    {
+      "name": "database",
+      "kind": "string",
+      "label": "Database",
+      "description": "The initial database for the Snowflake session."
+    },
+    {
+      "name": "default_target_schema",
+      "kind": "string",
+      "label": "Default Target Schema",
+      "description": "The default target database schema name to use for all streams."
+    },
+    {
+      "name": "faker_config.locale",
+      "kind": "array",
+      "label": "Faker Locale",
+      "description": "One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization"
+    },
+    {
+      "name": "faker_config.seed",
+      "kind": "string",
+      "label": "Faker Seed",
+      "description": "Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator"
+    },
+    {
+      "name": "flattening_enabled",
+      "kind": "boolean",
+      "label": "Enable Schema Flattening",
+      "description": "'True' to enable schema flattening and automatically expand nested properties."
+    },
+    {
+      "name": "flattening_max_depth",
+      "kind": "integer",
+      "label": "Max Flattening Depth",
+      "description": "The max depth to flatten schemas."
+    },
+    {
+      "name": "hard_delete",
+      "kind": "boolean",
+      "value": false,
+      "label": "Hard Delete",
+      "description": "Hard delete records."
+    },
+    {
+      "name": "load_method",
+      "kind": "options",
+      "value": "append-only",
+      "label": "Load Method",
+      "description": "The method to use when loading data into the destination. `append-only` will always write all input records whether that records already exists or not. `upsert` will update existing records and insert new records. `overwrite` will delete all existing records and insert all input records.",
+      "options": [
+        {
+          "label": "Append Only",
+          "value": "append-only"
+        },
+        {
+          "label": "Upsert",
+          "value": "upsert"
+        },
+        {
+          "label": "Overwrite",
+          "value": "overwrite"
+        }
+      ]
+    },
+    {
+      "name": "password",
+      "kind": "string",
+      "label": "Password",
+      "description": "The password for your Snowflake user.",
+      "sensitive": true
+    },
+    {
+      "name": "private_key",
+      "kind": "string",
+      "label": "Private Key",
+      "description": "The private key contents. For KeyPair authentication either private_key or private_key_path must be provided.",
+      "sensitive": true
+    },
+    {
+      "name": "private_key_passphrase",
+      "kind": "string",
+      "label": "Private Key Passphrase",
+      "description": "Passphrase to decrypt private key if encrypted.",
+      "sensitive": true
+    },
+    {
+      "name": "private_key_path",
+      "kind": "string",
+      "label": "Private Key Path",
+      "description": "Path to file containing private key. For KeyPair authentication either private_key or private_key_path must be provided.",
+      "sensitive": true
+    },
+    {
+      "name": "process_activate_version_messages",
+      "kind": "boolean",
+      "value": true,
+      "label": "Process `ACTIVATE_VERSION` messages",
+      "description": "Whether to process `ACTIVATE_VERSION` messages."
+    },
+    {
+      "name": "role",
+      "kind": "string",
+      "label": "Role",
+      "description": "The initial role for the session."
+    },
+    {
+      "name": "schema",
+      "kind": "string",
+      "label": "Schema",
+      "description": "The initial schema for the Snowflake session."
+    },
+    {
+      "name": "stream_map_config",
+      "kind": "object",
+      "label": "User Stream Map Configuration",
+      "description": "User-defined config values to be used within map expressions."
+    },
+    {
+      "name": "stream_maps",
+      "kind": "object",
+      "label": "Stream Maps",
+      "description": "Config object for stream maps capability. For more information check out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html)."
+    },
+    {
+      "name": "use_browser_authentication",
+      "kind": "boolean",
+      "value": false,
+      "label": "Use Browser Authentication",
+      "description": "Whether to use SSO authentication using an external browser."
+    },
+    {
+      "name": "user",
+      "kind": "string",
+      "label": "User",
+      "description": "The login name for your Snowflake user."
+    },
+    {
+      "name": "validate_records",
+      "kind": "boolean",
+      "value": true,
+      "label": "Validate Records",
+      "description": "Whether to validate the schema of the incoming streams."
+    },
+    {
+      "name": "warehouse",
+      "kind": "string",
+      "label": "Warehouse",
+      "description": "The initial warehouse for the session."
+    }
+  ]
+}