Skip to content

Commit d052d65

Browse files
colmsnowplowcolmsnowplow
authored
Merge pull request #132 from snowplow/release/web/redshift/1.3.0
Release/web/redshift/1.3.0
2 parents eaeb04a + d0ce9ec commit d052d65

34 files changed

+131
-53
lines changed

‎.test/great_expectations/expectations/web/v1/base_redshift.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@
286286
"meta": {
287287
"versions": {
288288
"test_suite_version": "1.1.1",
289-
"redshift_model_version": "1.2.0"
289+
"redshift_model_version": "1.3.0"
290290
},
291291
"great_expectations.__version__": "0.12.0"
292292
}

‎.test/great_expectations/expectations/web/v1/metadata.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@
103103
"meta": {
104104
"versions": {
105105
"test_suite_version": "1.1.1",
106-
"redshift_model_version": "1.2.0",
106+
"redshift_model_version": "1.3.0",
107107
"bigquery_model_version": "1.0.3",
108108
"snowflake_model_version": "1.0.1"
109109
},

‎.test/great_expectations/expectations/web/v1/page_view_in_session_values.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
"meta": {
2828
"versions": {
2929
"test_suite_version": "1.1.1",
30-
"redshift_model_version": "1.2.0",
30+
"redshift_model_version": "1.3.0",
3131
"bigquery_model_version": "1.0.3",
3232
"snowflake_model_version": "1.0.1"
3333
},

‎.test/great_expectations/expectations/web/v1/page_views.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@
225225
"meta": {
226226
"versions": {
227227
"test_suite_version": "1.1.1",
228-
"redshift_model_version": "1.2.0",
228+
"redshift_model_version": "1.3.0",
229229
"bigquery_model_version": "1.0.3",
230230
"snowflake_model_version": "1.0.1"
231231
},

‎.test/great_expectations/expectations/web/v1/sessions.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@
181181
"meta": {
182182
"versions": {
183183
"test_suite_version": "1.1.1",
184-
"redshift_model_version": "1.2.0",
184+
"redshift_model_version": "1.3.0",
185185
"bigquery_model_version": "1.0.3",
186186
"snowflake_model_version": "1.0.1"
187187
},

‎.test/great_expectations/expectations/web/v1/users.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@
117117
"meta": {
118118
"versions": {
119119
"test_suite_version": "1.1.1",
120-
"redshift_model_version": "1.2.0",
120+
"redshift_model_version": "1.3.0",
121121
"bigquery_model_version": "1.0.3",
122122
"snowflake_model_version": "1.0.1"
123123
},

‎CHANGELOG

+29-24
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,47 @@
1+
Redshift Web Version 1.3.0 (2022-06-06)
2+
--------------------------
3+
Redshift web: Change SORTKEY encoding to RAW (#129) (thanks @mark-walle!)
4+
Redshift Web: Fix column lengths in manifest tables (#131)
5+
16
Snowflake Web Version 1.0.1 (2022-02-24)
27
---------------------------------------
3-
Snowflake Web: Update column check stored procedure (Close #125)
4-
Snowflake Web: Fix varchar length for pseudonymized fields (Close #122)
5-
Snowflake Web: Remove start_date variable from users module (Close #123)
6-
Snowflake Web: Update copyright notices (Close #124)
7-
Snowflake Web: Fix logic in users_sessions_this_run to account for sparse data (Close #120)
8-
Snowflake Web: Fix varchar length for yauaa columns (Close #97)
9-
Snowflake Web: Fix se_label column length in events_staged (Close #109)
8+
Snowflake Web: Update column check stored procedure (#125)
9+
Snowflake Web: Fix varchar length for pseudonymized fields (#122)
10+
Snowflake Web: Remove start_date variable from users module (#123)
11+
Snowflake Web: Update copyright notices (#124)
12+
Snowflake Web: Fix logic in users_sessions_this_run to account for sparse data (#120)
13+
Snowflake Web: Fix varchar length for yauaa columns (#97)
14+
Snowflake Web: Fix se_label column length in events_staged (#109)
1015

1116
Snowflake Mobile Version 1.1.0 (2021-06-16)
1217
---------------------------------------
13-
Snowflake Mobile: Update update_manifest to throw error message (Close #104)
14-
Snowflake Mobile: Update commit_table procedure to throw error messages (Close #103)
15-
Snowflake Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #102)
18+
Snowflake Mobile: Update update_manifest to throw error message (#104)
19+
Snowflake Mobile: Update commit_table procedure to throw error messages (#103)
20+
Snowflake Mobile: Fix column order in events_this_run to allow for events_staged table migration (#102)
1621

1722
BigQuery Mobile Version 1.1.0 (2021-06-15)
1823
---------------------------------------
19-
BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (Close #100)
20-
BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #99)
24+
BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (#100)
25+
BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (#99)
2126

2227
Redshift Mobile Version 1.1.0 (2021-05-17)
2328
---------------------------------------
24-
Redshift Mobile: Increase session_id character limit in manifest (Close #93)
25-
Redshift Mobile: Add simplified configs (Close #89)
26-
Redshift Mobile: Set app errors module to disabled by default (Close #88)
27-
Redshift Mobile: Add model_tstamp to derived tables (Close #82)
28-
Redshift Mobile: Move app errors columns to end of table (Close #83)
29-
Redshift Mobile: Remove CTE from user-aggs (Close #80)
29+
Redshift Mobile: Increase session_id character limit in manifest (#93)
30+
Redshift Mobile: Add simplified configs (#89)
31+
Redshift Mobile: Set app errors module to disabled by default (#88)
32+
Redshift Mobile: Add model_tstamp to derived tables (#82)
33+
Redshift Mobile: Move app errors columns to end of table (#83)
34+
Redshift Mobile: Remove CTE from user-aggs (#80)
3035

3136
Snowflake Mobile Version 1.0.0 (2021-05-06)
3237
---------------------------------------
33-
Add Snowflake mobile model v1 (Close #85)
38+
Add Snowflake mobile model v1 (#85)
3439

3540
BigQuery Mobile Version 1.0.0 (2021-04-26)
3641
---------------------------------------
37-
Update licence (Close #79)
38-
Fix credential loading in scripts (close #72)
39-
Mobile: Add BigQuery mobile model v1 (close #73)
42+
Update licence (#79)
43+
Fix credential loading in scripts (#72)
44+
Mobile: Add BigQuery mobile model v1 (#73)
4045

4146
Redshift Mobile Version 1.0.0 (2021-03-25)
4247
---------------------------------------
@@ -51,8 +56,8 @@ Update Architecture diagram to match snowplow/snowplow (#62)
5156

5257
BigQuery Web Version 1.0.2 (2021-03-11)
5358
---------------------------------------
54-
BigQuery: Handle schema evolution in core contexts (close #52)
55-
Update SQL Runner download link to point to Github releases (close #51)
59+
BigQuery: Handle schema evolution in core contexts (#52)
60+
Update SQL Runner download link to point to Github releases (#51)
5661

5762
BigQuery Web Version 1.0.1 (2021-02-24)
5863
---------------------------------------

‎web/v1/redshift/CHANGELOG

+5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
Version 1.3.0 (2022-06-06)
2+
--------------------------
3+
Redshift web: Change SORTKEY encoding to RAW (#129) (thanks @mark-walle!)
4+
Redshift Web: Fix column lengths in manifest tables (#131)
5+
16
Version 1.2.0 (2021-01-25)
27
---------------------------
38
Redshift: Add config (#12)

‎web/v1/redshift/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ Detail on configuring the users module's playbook can be found [in the relevant
167167

168168
## A note on duplicates
169169

170-
This version of the model (1.2.0) excludes duplicated event_ids and page_view_ids. Ideally in the future it will provide standard options for handling them, however because customisations of this model involve queries on federated atomic tables, the safest option for a first version is to exclude them.
170+
This version of the model (>1.1.0) excludes duplicated event_ids and page_view_ids. Ideally in the future it will provide standard options for handling them, however because customisations of this model involve queries on federated atomic tables, the safest option for a first version is to exclude them.
171171

172172
If there is a need to handle duplicates, this can be done by adding a custom module to the base level of aggregation - take good care to manage the possibility of introducing duplicates downstream if doing so.
173173

‎web/v1/redshift/sql-runner/configs/migrations.json

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0",
3+
"data": {
4+
"enabled": true,
5+
"storage": "Default",
6+
"playbooks": [
7+
{
8+
"playbook": "standard/99-migrations/1.3.0-migration",
9+
"dependsOn": []
10+
}
11+
],
12+
"lockType": "hard",
13+
"owners": [
14+
]
15+
}
16+
}
17+

‎web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:input_schema: atomic
1313
:scratch_schema: scratch
1414
:output_schema: derived

‎web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:input_schema: atomic
1313
:scratch_schema: scratch
1414
:output_schema: derived

‎web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
:password:
99
:ssl:
1010
:variables:
11-
:model_version: redshift/web/1.2.0
11+
:model_version: redshift/web/1.3.0
1212
:scratch_schema: scratch
1313
:output_schema: derived
1414
:entropy: ""

‎web/v1/redshift/sql-runner/playbooks/standard/99-migrations/1.3.0-migration.yml.tmpl

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
:targets:
2+
- :name:
3+
:type: redshift
4+
:host:
5+
:database:
6+
:port:
7+
:username:
8+
:password:
9+
:ssl:
10+
:variables:
11+
:scratch_schema: scratch
12+
:output_schema: derived
13+
:entropy: ""
14+
:steps:
15+
- :name: 1.3.0-migration
16+
:queries:
17+
- :name: base-session-id-manifest
18+
:file: standard/99-migrations/1.3.0-migration/base-session-id-manifest.sql
19+
:template: true
20+
- :name: 1.3.0-migration
21+
:queries:
22+
- :name: sessions-userid-manifest
23+
:file: standard/99-migrations/1.3.0-migration/sessions-userid-manifest.sql
24+
:template: true
25+
- :name: 1.3.0-migration
26+
:queries:
27+
- :name: users-manifest
28+
:file: standard/99-migrations/1.3.0-migration/users-manifest.sql
29+
:template: true
30+
- :name: 1.3.0-migration
31+
:queries:
32+
- :name: sortkey-encoding
33+
:file: standard/99-migrations/1.3.0-migration/sortkey-encoding.sql
34+
:template: true

‎web/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql

+2-2
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ INSERT INTO {{.output_schema}}.base_event_id_manifest{{.entropy}} (
108108
);
109109

110110
CREATE TABLE IF NOT EXISTS {{.output_schema}}.base_session_id_manifest{{.entropy}} (
111-
session_id VARCHAR(36),
111+
session_id VARCHAR(128),
112112
min_tstamp TIMESTAMP
113113
)
114114
DISTSTYLE KEY
@@ -117,7 +117,7 @@ SORTKEY (min_tstamp);
117117

118118
INSERT INTO {{.output_schema}}.base_session_id_manifest{{.entropy}} (
119119
SELECT
120-
'seed'::VARCHAR(36),
120+
'seed'::VARCHAR(128),
121121
'{{.start_date}}'::TIMESTAMP
122122

123123
WHERE

‎web/v1/redshift/sql-runner/sql/standard/02-page-views/01-main/00-setup-page-views.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.page_views{{.entropy}} (
106106
dvce_created_tstamp TIMESTAMP ENCODE ZSTD,
107107
collector_tstamp TIMESTAMP ENCODE ZSTD,
108108
derived_tstamp TIMESTAMP ENCODE ZSTD,
109-
start_tstamp TIMESTAMP ENCODE ZSTD,
109+
start_tstamp TIMESTAMP ENCODE RAW,
110110
end_tstamp TIMESTAMP ENCODE ZSTD,
111111

112112
engaged_time_in_s INT ENCODE ZSTD,

‎web/v1/redshift/sql-runner/sql/standard/03-sessions/01-main/00-setup-sessions.sql

+2-2
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.sessions{{.entropy}} (
9595
domain_sessionid VARCHAR(128) ENCODE ZSTD,
9696
domain_sessionidx INT ENCODE ZSTD,
9797

98-
start_tstamp TIMESTAMP ENCODE ZSTD,
98+
start_tstamp TIMESTAMP ENCODE RAW,
9999
end_tstamp TIMESTAMP ENCODE ZSTD,
100100

101101
-- user fields
@@ -220,7 +220,7 @@ SORTKEY (start_tstamp);
220220

221221
-- Staged manifest table as input to users step
222222
CREATE TABLE IF NOT EXISTS {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}} (
223-
domain_userid VARCHAR(36),
223+
domain_userid VARCHAR(128),
224224
start_tstamp TIMESTAMP
225225
)
226226
DISTSTYLE KEY

0 commit comments

Comments
 (0)