Skip to content

Commit 2343b1d

Browse files
chuwychuwy
committed
Merge branch 'release/r28'
2 parents 15caf98 + 66e2aff commit 2343b1d

34 files changed

+909
-209
lines changed

‎.travis.yml

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,22 @@ scala:
44
jdk:
55
- oraclejdk8
66
before_deploy: pip install --user release-manager==0.3.0
7-
deploy:
8-
- provider: script
9-
script: release-manager --config .travis/release.yml --make-artifact --upload-artifact --check-version
10-
skip_cleanup: true
11-
on:
12-
tags: true
137
env:
148
global:
159
- secure: my2pgROi/Xut3sVM47URmR+A/e29Z+pW3dtcHfqJHn3c6CP1NaSSKM+6e3PdM7BYNl9HgT0zUYfnw4lstIi9nCP0vumcA4zbx+zOd9brV10/jhpO/RtMsSQB1ycpGffJP5z+sc6AzK11ECkOoXlGHZNx48Lf9SaDTL79tMn5MQOiJphy+4OKYBiyZEdiPX6C45uvItG9zMN6RhlVpryHzRwdeiqykdFUCd/SeKAiXnAnNFPUyEYAlKz9P3setkZKVzp/yKBNVK41W1myePOWB0Is0MSisOapf1MD/k+tXovH9Ilo+K+HZId5hMpGQMo5tCvb2N2xr7j+VsNwcW3kjgROH/0QfJD9l5pB7P67iKv0JPrOjGpI2DgO7o4/YspAYUferWEeg3JboF+eytSJgXkDF6y4Ll5xQEhEXF5VDZfKamg2MOQBnRa13egn9oom5KcQ/7UrxyirRUFQI04fp3M8hGuYjxIy98t3/7Zw448ayj+oJd0miz449/3iz/YZfuaqlTFHxc22DL9BG9hX7ygOitDDoacGEeK0Z5iL7samU8GI+V5aJP23SaV+t1gvutCEU4NwRGa+ZTPmEt4RInEko0kF69gtGEHq026vjvBGW8uKFcyZcrq1ZJTcRHo3oj2L3Ub1KKuyTxs78Jd3dAI59DSG5ZSkACcuVt5gQdI=
1610
- secure: Q/H5Mcb1PxmpyJ8wgCWgnof098r3OA9iVqQiySb+TQSnEjv1FKfvzMU1X3HK2TePwG5SPI4QoXygsXnDuo2lV4cPtofwULDvtNoqd8ii2ojFHbp6nvSrD7fxSVq0Vr20FfavrWzqN9/dB8m7/vyuLl2DGTsSFNMrB40Gt1OfoYFvnah3RvyhFYw0WNluPp3n4+l1zd4I64WJKU60zP2mvzRaNzXPfvXf9igGbC9vtIxaaVb9jRr7DtQrk3wqxdjY3t5Zr6ZDYtqFeUkOCskIb0ghL4Idegm7D8MMKemz8N+ZMZPQgJkGMZQj4gxBwFH0YtROnx1y4O8c21CwKA8LVg1H2n7nrwEHyjaZIwCbZbT8eoYHyyvTeXhhMFTsk/GCksMWvV4K9SuvSsDz33yDPsopdEdwfGJ6F4wqhb5MezjSlktaGeAEGsp04H0AXjprTumCboodYAgLz3CZuv0OtMyxHyH+V2q2hLnhJiWC5ue0WsWLx1WzfzIhHIxUvVqtY4fHyZW3OBycvbvDm/Auxj8xkWAXzBP315vJSyFLHkde3VThPnQC17zdN4JBcyeGXVNd7ZbIvoEQfjKOUmaW1plFIZtQPyqfx6vXWak/6ICI6hX+lAgSBjMHkvv/4lGrbgQOrM+VzkfRBkVb3Ybi+IDBkE5wodAntB5KQG9GsSA=
1711
- secure: MJp/yUl9TViL4mCMu5v2gNuxvrfG5/atcOMjNchtvs3fzY9RNEMp0d0Qj2HVzSdTY0HizWlprc9vOWtiMeoDnGaMNbPQSxNoujnTq+xK4U6HycFuD1iGGTr/Zf5NfGAmQP9ty9CgHZJkXKrKHIlw2fJV18vq82fykiOkb+lFXhrSd373gO6tmG9AQsfd/+DTTpGVE9yblfFyqa8Zed59Qrr9ZC2/qlJwouFKDjIOc/iU0rwFR+MaxiaezLIOyPNT/K/zXyoiNQwr3uMv3EZqvO1yv/ymW3y6sk1IqyWzgP9fjFllHdhGOPLoUMSx898NihaCeXUsMZeFEFQ95sgevPiRpOJ47qd1Qimx8+o8Q0kTqU9evY9wo+Rsd8Odqx1+MIPabk1xzDB4BvDMqsHQfRx0CJT82sfW+fXxoHNTFU7I7jJBBL5T7GdntXSMxFZuYL34TFTiEkUBz5e/k6oY5c987iU4EX5Jbgq9sWbvJ6UTLqemMBMIR4TMaYlWNpXp5bHnsaBQqlS1VzkF/PWnx8edVVvfk7CW27b9asnEvBYeM62XibSihGbRqydv8QU3OFjH3mq4bZAOc/ePlnky3OOCiXKhh2PJusQpZa6PrRsDE+/4q7jkebLiC16S9Dah1FZeqzGJDXMabH4WGG4BdDflgrYueMKA8HnH2rtLvMA=
1812
- secure: sP7R6sZqekVEeAt/NfgTO9kLZ+tpWrVCRWJJkulh2cW6flPkGqnSH+5HHa8dj00ZNojl7NZqyIwnmBOCfkYS+ncIpXNvHah2DLEuwZqxXtZhLU2mZg6DKGBaCfArhtXRf6Y8FQ+e9unZi9A5yqT7HScaABeWglYCJIIZ6o0tFL9MB0Zx6b1bHDAgrsvFCqUeyWvt7lEQ8fHi/pgHaVuXAmyO2bYKgY2Vwb6bILUTtFMSvknfIXPzdUNQeOalGXUjHGtsSvyuKZbR8kOhnAn9fjgNtxIyUT+u1BUY0HfuYH8653o8XzQMdifja9C0LPAoJ+eaCBQOXV/ZxoXr1/deNPZfLsaIXtmmzXpHbniWDYuM69zvrZM0y+VsuhIOCKVtllvyvd3EGNwjIP76HYTYxCeGh0p3R7Vde2BgsBBf45VkGTrNKA7sOyZ8JilaGnlEfB7ujyZ6L1QXd1ihRryuoZ8iXzngxobKmqC9m+uCyeC6+cvU2jeNVTkYfoVaRW3zw6RgSDLG5mJfZPREwOBCRQXms6ifzgj7HFw1W5FdlHrM+EyvTznnXWk1OGFeIVXezqtCIq8d9TpRQbG1ezlK0y/TYasidUrJUo2tri4LFTkkGyqUSQxNU+Tkjm18nUUjlphPLMANO6uuD1E6mouG7nQOo/ipR/46jUCPCgcrl20=
13+
deploy:
14+
- provider: script
15+
script: ./.travis/deploy.sh rdb_loader $TRAVIS_TAG
16+
skip_cleanup: true
17+
on:
18+
tags: true
19+
condition: '"$(.travis/is_release_tag.sh rdb_loader $TRAVIS_TAG)" == "" && $? == 0'
20+
- provider: script
21+
script: ./.travis/deploy.sh rdb_shredder $TRAVIS_TAG
22+
skip_cleanup: true
23+
on:
24+
tags: true
25+
condition: '"$(.travis/is_release_tag.sh rdb_shredder $TRAVIS_TAG)" == "" && $? == 0'

‎.travis/deploy.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
project=$1
4+
tag=$2
5+
6+
slashed="${project}/"
7+
slashed_len=${#slashed}
8+
9+
cicd=${tag:0:${slashed_len}}
10+
release=${tag:${slashed_len}}
11+
12+
if [ "${cicd}" == "${slashed}" ]; then
13+
if [ "${release}" == "" ]; then
14+
echo "WARNING! No release specified! Ignoring."
15+
exit 2
16+
fi
17+
else
18+
echo "This can't be deployed - there's no ${project} tag! (Is the travis condition set?)"
19+
exit 1
20+
fi
21+
22+
cd "${TRAVIS_BUILD_DIR}"
23+
24+
export TRAVIS_BUILD_RELEASE_TAG="${release}"
25+
release-manager \
26+
--config "./.travis/release_${project}.yml" \
27+
--check-version \
28+
--make-version \
29+
--make-artifact \
30+
--upload-artifact

‎.travis/is_release_tag.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
3+
project=$1
4+
tag=$2
5+
6+
slashed="${project}/"
7+
slashed_len=${#slashed}
8+
9+
cicd=${tag:0:${slashed_len}}
10+
release=${tag:${slashed_len}}
11+
12+
if [ "${cicd}" == "${slashed}" ]; then
13+
if [ "${release}" == "" ]; then
14+
echo "Warning! No release specified! Ignoring."
15+
exit 2
16+
fi
17+
exit 0
18+
else
19+
exit 1
20+
fi

‎.travis/release.yml renamed to ‎.travis/release_rdb_loader.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ packages:
6464
version : <%= FUNC['sbt_version(.)'] %>
6565

6666
# Required IF '--check-version' is passed: will assert that both versions are the same
67-
build_version : <%= ENV['TRAVIS_TAG'] %>
67+
build_version : <%= ENV['TRAVIS_BUILD_RELEASE_TAG'] %>
6868

6969
# Optional: Build commands
7070
build_commands:

‎.travis/release_rdb_shredder.yml

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
---
2+
# --- Variables --- #
3+
4+
local:
5+
root_dir : <%= ENV['TRAVIS_BUILD_DIR'] %>
6+
7+
# --- Release Manager Config --- #
8+
9+
# Required: deployment targets
10+
targets:
11+
- type : "awss3"
12+
access_key_id : <%= ENV['AWS_ACCESS_KEY'] %>
13+
secret_access_key : <%= ENV['AWS_SECRET_KEY'] %>
14+
15+
# Required: packages to be deployed
16+
packages:
17+
- name : "rdb-shredder-hosted-asset"
18+
locations:
19+
- bucket : "snowplow-hosted-assets"
20+
path : "4-storage/rdb-shredder"
21+
region : "eu-west-1"
22+
- bucket : "snowplow-hosted-assets-us-east-1"
23+
path : "4-storage/rdb-shredder"
24+
region : "us-east-1"
25+
- bucket : "snowplow-hosted-assets-us-west-1"
26+
path : "4-storage/rdb-shredder"
27+
region : "us-west-1"
28+
- bucket : "snowplow-hosted-assets-us-west-2"
29+
path : "4-storage/rdb-shredder"
30+
region : "us-west-2"
31+
- bucket : "snowplow-hosted-assets-sa-east-1"
32+
path : "4-storage/rdb-shredder"
33+
region : "sa-east-1"
34+
- bucket : "snowplow-hosted-assets-eu-central-1"
35+
path : "4-storage/rdb-shredder"
36+
region : "eu-central-1"
37+
- bucket : "snowplow-hosted-assets-ap-southeast-1"
38+
path : "4-storage/rdb-shredder"
39+
region : "ap-southeast-1"
40+
- bucket : "snowplow-hosted-assets-ap-southeast-2"
41+
path : "4-storage/rdb-shredder"
42+
region : "ap-southeast-2"
43+
- bucket : "snowplow-hosted-assets-ap-northeast-1"
44+
path : "4-storage/rdb-shredder"
45+
region : "ap-northeast-1"
46+
- bucket : "snowplow-hosted-assets-ap-south-1"
47+
path : "4-storage/rdb-shredder"
48+
region : "ap-south-1"
49+
- bucket : "snowplow-hosted-assets-us-east-2"
50+
path : "4-storage/rdb-shredder"
51+
region : "us-east-2"
52+
- bucket : "snowplow-hosted-assets-ca-central-1"
53+
path : "4-storage/rdb-shredder"
54+
region : "ca-central-1"
55+
- bucket : "snowplow-hosted-assets-eu-west-2"
56+
path : "4-storage/rdb-shredder"
57+
region : "eu-west-2"
58+
- bucket : "snowplow-hosted-assets-ap-northeast-2"
59+
path : "4-storage/rdb-shredder"
60+
region : "ap-northeast-2"
61+
publish : true
62+
override : false
63+
continue_on_conflict : false
64+
version : <%= CMD['sbt "project shredder" version -Dsbt.log.noformat=true | awk "END{print}" | sed -r "s/\[info\]\s(.*)/\1/" | xargs echo -n'] %>
65+
66+
# Required IF '--check-version' is passed: will assert that both versions are the same
67+
build_version : <%= ENV['TRAVIS_BUILD_RELEASE_TAG'] %>
68+
69+
# Optional: Build commands
70+
build_commands:
71+
- sbt "project shredder" assembly
72+
73+
# Required: Artifact. Version will be inserted between prefix and suffix
74+
artifacts:
75+
- type : "asis"
76+
prefix : "snowplow-rdb-shredder-"
77+
suffix : ".jar"
78+
binary_paths:
79+
- "shredder/target/scala-2.11/snowplow-rdb-shredder-{{ packages.0.build_version }}.jar"

‎CHANGELOG

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,23 @@
1+
Release 28 (2017-11-13)
2+
-----------------------
3+
Common: add CI/CD (#55)
4+
Common: remove AWS Java SDK shading (#54)
5+
RDB Shredder: add Snowplow and Clojars resolvers (#56)
6+
RDB Shredder: bump Spark to 2.2.0 (#52)
7+
RDB Shredder: bump to 0.13.0 (#49)
8+
RDB Shredder: bump scala-common-enrich to 0.27.0 (#39)
9+
RDB Shredder: overwrite output datasets (#41)
10+
RDB Loader: bump sbt-assembly to 0.14.5 (#51)
11+
RDB Loader: bump SBT to 0.13.16 (#50)
12+
RDB Loader: allow JDBC credentials to be stored in EC2 parameter store (#19)
13+
RDB Loader: add support for SSH tunnels (#22)
14+
RDB Loader: bump AWS SDK to 1.11.208 (#48)
15+
RDB Loader: bump redshift-jdbc to 1.2.8.1005 (#40)
16+
RDB Loader: make loading shredded data always required (#29)
17+
RDB Loader: remove tracking from dry run (#42)
18+
RDB Loader: execute manifest insert in same transaction as load (#36)
19+
RDB Loader: make logkey optional (#35)
20+
121
Version 0.13.0 (2017-09-06)
222
---------------------------
323
Common: migrate CHANGELOG from snowplow/snowplow (#24)

‎build.sbt

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
lazy val loader = project.in(file("."))
1515
.settings(
1616
name := "snowplow-rdb-loader",
17-
version := "0.13.0",
17+
version := "0.14.0",
1818
initialCommands := "import com.snowplowanalytics.snowplow.rdbloader._",
1919
mainClass in Compile := Some("com.snowplowanalytics.snowplow.rdbloader.Main")
2020
)
@@ -37,7 +37,10 @@ lazy val loader = project.in(file("."))
3737

3838
Dependencies.postgres,
3939
Dependencies.redshift,
40+
Dependencies.redshiftSdk,
4041
Dependencies.s3,
42+
Dependencies.ssm,
43+
Dependencies.jSch,
4144

4245
Dependencies.specs2,
4346
Dependencies.specs2ScalaCheck,
@@ -48,7 +51,7 @@ lazy val loader = project.in(file("."))
4851
lazy val shredder = project.in(file("shredder"))
4952
.settings(
5053
name := "snowplow-rdb-shredder",
51-
version := "0.12.0",
54+
version := "0.13.0",
5255
description := "Spark job to shred event and context JSONs from Snowplow enriched events",
5356
BuildSettings.oneJvmPerTestSetting // ensures that only CrossBatchDeduplicationSpec has a DuplicateStorage
5457
)

‎project/BuildSettings.scala

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,14 @@ object BuildSettings {
6464

6565
lazy val assemblySettings = Seq(
6666
jarName,
67+
68+
assemblyShadeRules in assembly := Seq(
69+
ShadeRule.rename(
70+
// EMR has 0.1.42 installed
71+
"com.jcraft.jsch.**" -> "shadejsch.@1"
72+
).inAll
73+
),
74+
6775
assemblyMergeStrategy in assembly := {
6876
case PathList("META-INF", _ @ _*) => MergeStrategy.discard
6977
case PathList("reference.conf", _ @ _*) => MergeStrategy.concat
@@ -73,14 +81,6 @@ object BuildSettings {
7381

7482
lazy val shredderAssemblySettings = Seq(
7583
jarName,
76-
// Slightly cleaner jar name
77-
// For AMI 4.5.0, could be removed in future versions
78-
assemblyShadeRules in assembly := Seq(
79-
ShadeRule.rename(
80-
"com.amazonaws.**" -> "shadeaws.@1",
81-
"org.apache.http.**" -> "shadehttp.@1"
82-
).inAll
83-
),
8484
// Drop these jars
8585
assemblyExcludedJars in assembly := {
8686
val cp = (fullClasspath in assembly).value

‎project/Dependencies.scala

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,14 @@ object Dependencies {
2626
val cats = "0.9.0"
2727

2828
// Scala (Shredder)
29-
val spark = "2.1.0"
30-
val commonEnrich = "0.25.0"
29+
val spark = "2.2.0"
30+
val commonEnrich = "0.27.0"
3131

3232
// Java (Loader)
3333
val postgres = "42.0.0"
34-
val redshift = "1.2.1.1001"
35-
val aws = "1.10.77"
34+
val redshift = "1.2.8.1005"
35+
val aws = "1.11.208"
36+
val jSch = "0.1.54"
3637

3738
// Java (Shredder)
3839
val dynamodb = "1.11.98"
@@ -47,7 +48,11 @@ object Dependencies {
4748
// For specs2
4849
"scalaz-bintray" at "http://dl.bintray.com/scalaz/releases",
4950
// Redshift native driver
50-
"redshift" at "http://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release"
51+
"redshift" at "http://redshift-maven-repository.s3-website-us-east-1.amazonaws.com/release",
52+
// For Snowplow libs (SCE transient)
53+
"Snowplow Analytics Maven repo" at "http://maven.snplow.com/releases/",
54+
// For uaParser utils (SCE transient)
55+
"user-agent-parser repo" at "https://clojars.org/repo/"
5156
)
5257

5358
// Scala (Loader)
@@ -70,9 +75,12 @@ object Dependencies {
7075
val sparkSQL = "org.apache.spark" %% "spark-sql" % V.spark % "provided"
7176

7277
// Java (Loader)
73-
val postgres = "org.postgresql" % "postgresql" % V.postgres
74-
val redshift = "com.amazon.redshift" % "redshift-jdbc42" % V.redshift
75-
val s3 = "com.amazonaws" % "aws-java-sdk-s3" % V.aws
78+
val postgres = "org.postgresql" % "postgresql" % V.postgres
79+
val redshift = "com.amazon.redshift" % "redshift-jdbc42-no-awssdk" % V.redshift
80+
val redshiftSdk = "com.amazonaws" % "aws-java-sdk-redshift" % V.aws
81+
val s3 = "com.amazonaws" % "aws-java-sdk-s3" % V.aws
82+
val ssm = "com.amazonaws" % "aws-java-sdk-ssm" % V.aws
83+
val jSch = "com.jcraft" % "jsch" % V.jSch
7684

7785
// Java (Shredder)
7886
val dynamodb = "com.amazonaws" % "aws-java-sdk-dynamodb" % V.dynamodb

‎project/build.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
sbt.version=0.13.15
1+
sbt.version=0.13.16

‎project/plugins.sbt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
logLevel := Level.Warn
22

3-
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4")
3+
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.5")

‎shredder/src/main/scala/com.snowplowanalytics.snowplow.storage/spark/ShredJob.scala

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ import com.fasterxml.jackson.databind.JsonNode
2727
// Spark
2828
import org.apache.spark.{SparkConf, SparkContext}
2929
import org.apache.spark.serializer.KryoSerializer
30-
import org.apache.spark.sql.SparkSession
30+
import org.apache.spark.sql.{Row, SaveMode, SparkSession}
31+
import org.apache.spark.sql.types.{StringType, StructField, StructType}
3132

3233
// Scalaz
3334
import scalaz._
@@ -64,7 +65,8 @@ object ShredJob extends SparkJob {
6465
classOf[com.fasterxml.jackson.databind.node.ArrayNode],
6566
classOf[com.fasterxml.jackson.databind.node.NullNode],
6667
classOf[com.fasterxml.jackson.databind.node.JsonNodeFactory],
67-
classOf[org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage]
68+
classOf[org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage],
69+
classOf[org.apache.spark.sql.execution.datasources.FileFormatWriter$WriteTaskResult]
6870
)
6971
override def sparkConfig(): SparkConf = new SparkConf()
7072
.setAppName(getClass().getSimpleName())
@@ -345,7 +347,7 @@ class ShredJob(@transient val spark: SparkSession, args: Array[String]) extends
345347
// Handling of malformed rows
346348
val bad = common
347349
.flatMap { case (line, shredded) => projectBads(line, shredded) }
348-
.map { case (line, errors) => new BadRow(line, errors).toCompactJson }
350+
.map { case (line, errors) => Row(BadRow(line, errors).toCompactJson) }
349351

350352
// Handling of properly-formed rows, only one event from an event id and event fingerprint
351353
// combination is kept
@@ -393,10 +395,13 @@ class ShredJob(@transient val spark: SparkSession, args: Array[String]) extends
393395
// Write errors unioned with errors occurred during cross-batch deduplication
394396
val badDupes = bad ++ dupeFailed
395397
.flatMap { case (s, dupe) => dupe match {
396-
case Failure(m) => Some(new BadRow(s.originalLine, m).toCompactJson)
398+
case Failure(m) => Some(Row(BadRow(s.originalLine, m).toCompactJson))
397399
case _ => None
398-
}}
399-
badDupes.saveAsTextFile(shredConfig.badFolder)
400+
} }
401+
spark.createDataFrame(badDupes, StructType(StructField("_", StringType, true) :: Nil))
402+
.write
403+
.mode(SaveMode.Overwrite)
404+
.text(shredConfig.badFolder)
400405

401406
// Create duplicate JSON contexts for well-formed events having duplicates
402407
// This creates not canonical contexts (with schema and data), but shredded hierarchies
@@ -407,8 +412,11 @@ class ShredJob(@transient val spark: SparkSession, args: Array[String]) extends
407412

408413
// Ready the events for database load
409414
val events = goodWithSyntheticDupes
410-
.map(e => alterEnrichedEvent(e.shredded.originalLine, e.newEventId))
411-
events.saveAsTextFile(getAlteredEnrichedOutputPath(shredConfig.outFolder))
415+
.map(e => Row(alterEnrichedEvent(e.shredded.originalLine, e.newEventId)))
416+
spark.createDataFrame(events, StructType(StructField("_", StringType, true) :: Nil))
417+
.write
418+
.mode(SaveMode.Overwrite)
419+
.text(getAlteredEnrichedOutputPath(shredConfig.outFolder))
412420

413421
// Update the shredded JSONs with the new deduplicated event IDs and stringify
414422
val jsons = (goodWithSyntheticDupes

0 commit comments

Comments
 (0)