Merge pull request #62 from dpguthrie/masking

add masking

Author: dpguthrie

dbt_project.yml

@@ -27,12 +27,17 @@ clean-targets:         # directories to be removed by `dbt clean`
     - "dbt_modules"
     - "dbt_packages"
 
+on-run-start:
+  - "{{ dbt_snow_mask.create_masking_policy('sources')}}"
+  - "{{ dbt_snow_mask.apply_masking_policy('sources')}}"
+
 vars:
   start_date: '1999-01-01'
   disable_dbt_artifacts_autoupload: "{% if env_var('DBT_TARGET_ENV') == 'prod' %}false{% else %}true{% endif %}"
   disable_run_results: "{% if env_var('DBT_TARGET_ENV') == 'prod' %}false{% else %}true{% endif %}"
   disable_tests_results: "{% if env_var('DBT_TARGET_ENV') == 'prod' %}false{% else %}true{% endif %}"
   disable_dbt_invocation_autoupload: "{% if env_var('DBT_TARGET_ENV') == 'prod' %}false{% else %}true{% endif %}"
+  use_force_applying_masking_policy: "True"
 
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models

dependencies.yml

@@ -12,3 +12,5 @@ packages:
     version: ["<3.0.0"]
   - package: elementary-data/elementary
     version: 0.12.1
+  - package: entechlog/dbt_snow_mask
+    version: 0.2.4

macros/masking/create_masking_policy_encrypt_pii.sql

@@ -0,0 +1,10 @@
+{% macro create_masking_policy_encrypt_pii(node_database, node_schema) %}
+
+CREATE MASKING POLICY IF NOT EXISTS {{ node_database }}.{{ node_schema }}.encrypt_pii AS (val string) 
+  RETURNS string ->
+      CASE
+        WHEN CURRENT_ROLE() IN ('TRANSFORMER') THEN val
+        ELSE SHA2(val)
+      END
+
+{% endmacro %}

models/staging/tpch/_tpch__models.yml

@@ -23,10 +23,6 @@ models:
         description: market segment of the customer
       - name: comment
         description: '{{ doc("comment") }}'
-    meta:
-      joins:
-        - join: stg_tpch_nations
-          sql_on: ${stg_tpch_customers.nation_key} = ${stg_tpch_nations.nation_key}
 
   - name: stg_tpch_line_items
     description: staging layer for line items data
@@ -83,10 +79,6 @@ models:
         description: foreign key to stg_tpch_regions
       - name: comment
         description: additional commentary
-    meta:
-      joins:
-        - join: stg_tpch_regions
-          sql_on: ${stg_tpch_nations.region_key} = ${stg_tpch_regions.region_key}
 
   - name: stg_tpch_orders
     description: staging layer for orders data
@@ -131,10 +123,6 @@ models:
         description: '{{ doc("cost") }}'
       - name: comment
         description: '{{ doc("comment") }}'
-    meta:
-      joins:
-        - join: stg_tpch_suppliers
-          sql_on: ${stg_tpch_part_suppliers.supplier_key} = ${stg_tpch_suppliers.supplier_key}
 
   - name: stg_tpch_parts
     description: staging layer for parts data
@@ -194,7 +182,3 @@ models:
         description: '{{ doc("account_balance") }}'
       - name: comment
         description: '{{ doc("comment") }}'
-    meta:
-      joins:
-        - join: stg_tpch_nations
-          sql_on: ${stg_tpch_suppliers.nation_key} = ${stg_tpch_nations.nation_key}

models/staging/tpch/_tpch__sources.yml

@@ -57,8 +57,12 @@ sources:
               - unique
           - name: c_name
             description: customer id
+            meta:
+              masking_policy: encrypt_pii
           - name: c_address
             description: address of the customer
+            meta:
+              masking_policy: encrypt_pii
           - name: c_nationkey
             description: foreign key to stg_tpch_nations
             tests:
@@ -67,6 +71,8 @@ sources:
                   field: n_nationkey
           - name: c_phone
             description: phone number of the customer
+            meta:
+              masking_policy: encrypt_pii
           - name: c_acctbal
             description: raw account balance 
           - name: c_mktsegment

package-lock.yml

@@ -9,4 +9,6 @@ packages:
   version: 2.6.2
 - package: elementary-data/elementary
   version: 0.12.1
-sha1_hash: 4602085df3a0e934c8834bc00192f0207fdaa3b7
+- package: entechlog/dbt_snow_mask
+  version: 0.2.4
+sha1_hash: 297b60f83c9161c116f2335d60f69f831892b5e1