Skip to content

Commit dc1a49c

Browse files
spenesspenes
committed
Implicit region config reader
1 parent 1069842 commit dc1a49c

File tree

15 files changed

+479
-116
lines changed

15 files changed

+479
-116
lines changed

‎config/config.kinesis.minimal.hocon

+4-3
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,20 @@
44
"type": "kinesis"
55
"streamName": "test-kinesis-stream"
66
"initialPosition": "LATEST"
7-
"region": "eu-central-1"
87
}
98
"output": {
109
"good": {
1110
"client": {
1211
"endpoint": "localhost"
1312
}
13+
"cluster": {
14+
"index": "good"
15+
}
1416
}
1517
"bad" {
1618
"type": "kinesis"
1719
"streamName": "test-kinesis-bad-stream"
18-
"region": "eu-central-1"
1920
}
2021
}
21-
"purpose": "BAD_ROWS"
22+
"purpose": "ENRICHED_EVENTS"
2223
}

‎config/config.nsq.minimal.hocon

+3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212
"client": {
1313
"endpoint": "localhost"
1414
}
15+
"cluster": {
16+
"index": "good"
17+
}
1518
}
1619
"bad" {
1720
"type": "nsq"

‎core/src/main/resources/application.conf

-3
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@
2424
"aws": {
2525
"signing": false
2626
}
27-
"cluster": {
28-
"index": "good"
29-
}
3027
}
3128
}
3229
"monitoring": {

‎core/src/main/scala/com.snowplowanalytics.stream/loader/Config.scala

+101-79
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,14 @@ import com.monovore.decline.{Command, Opts}
2828
import cats.syntax.either._
2929
import cats.syntax.validated._
3030

31-
import com.amazonaws.regions.DefaultAwsRegionProviderChain
31+
import com.amazonaws.regions.{DefaultAwsRegionProviderChain, Regions}
3232

33-
import pureconfig.{CamelCase, ConfigFieldMapping, ConfigObjectSource, ConfigReader, ConfigSource}
33+
import com.typesafe.config.ConfigOrigin
34+
35+
import pureconfig._
3436
import pureconfig.generic.{FieldCoproductHint, ProductHint}
3537
import pureconfig.generic.semiauto._
36-
import pureconfig.error.{ConfigReaderFailures, FailureReason}
38+
import pureconfig.error._
3739

3840
object Config {
3941

@@ -68,7 +70,7 @@ object Config {
6870
initialPosition: String,
6971
initialTimestamp: Option[String],
7072
maxRecords: Long,
71-
region: Option[String],
73+
region: Region,
7274
appName: String,
7375
customEndpoint: Option[String],
7476
dynamodbCustomEndpoint: Option[String],
@@ -91,15 +93,6 @@ object Config {
9193

9294
object Kinesis {
9395
final case class Buffer(byteLimit: Long, recordLimit: Long, timeLimit: Long)
94-
95-
implicit val sourceKinesisConfigReader: ConfigReader[Kinesis] =
96-
deriveReader[Kinesis].emap { c =>
97-
val region = c.region.orElse(getRegion)
98-
region match {
99-
case Some(_) => c.copy(region = region).asRight
100-
case _ => RawFailureReason("Region isn't set in the Kinesis source").asLeft
101-
}
102-
}
10396
}
10497
}
10598

@@ -131,17 +124,7 @@ object Config {
131124
ssl: Boolean
132125
)
133126

134-
final case class ESAWS(signing: Boolean, region: Option[String])
135-
object ESAWS {
136-
implicit val sinkGoodESAWSConfigReader: ConfigReader[ESAWS] =
137-
deriveReader[ESAWS].emap { c =>
138-
val region = c.region.orElse(getRegion)
139-
if (c.signing && region.isEmpty)
140-
RawFailureReason("Region needs to be set when AWS signing is true").asLeft
141-
else
142-
c.copy(region = region).asRight
143-
}
144-
}
127+
final case class ESAWS(signing: Boolean, region: Region)
145128

146129
final case class ESCluster(index: String, documentType: Option[String])
147130

@@ -163,19 +146,9 @@ object Config {
163146

164147
final case class Kinesis(
165148
streamName: String,
166-
region: Option[String],
149+
region: Region,
167150
customEndpoint: Option[String]
168151
) extends BadSink
169-
object Kinesis {
170-
implicit val sinkBadKinesisConfigReader: ConfigReader[Kinesis] =
171-
deriveReader[Kinesis].emap { c =>
172-
val region = c.region.orElse(getRegion)
173-
region match {
174-
case Some(_) => c.copy(region = region).asRight
175-
case _ => RawFailureReason("Region isn't set in the Kinesis sink").asLeft
176-
}
177-
}
178-
}
179152
}
180153
}
181154

@@ -208,48 +181,81 @@ object Config {
208181
final case class Metrics(cloudWatch: Boolean)
209182
}
210183

211-
final case class RawFailureReason(description: String) extends FailureReason
184+
final case class Region(name: String)
212185

213-
implicit val streamLoaderConfigReader: ConfigReader[StreamLoaderConfig] =
214-
deriveReader[StreamLoaderConfig]
215-
implicit val sourceConfigReader: ConfigReader[Source] =
216-
deriveReader[Source]
217-
implicit val sourceStdinConfigReader: ConfigReader[Source.Stdin.type] =
218-
deriveReader[Source.Stdin.type]
219-
implicit val sourceNsqConfigReader: ConfigReader[Source.Nsq] =
220-
deriveReader[Source.Nsq]
221-
implicit val sourceNsqBufferConfigReader: ConfigReader[Source.Nsq.Buffer] =
222-
deriveReader[Source.Nsq.Buffer]
223-
implicit val sourceKinesisConfigBufferReader: ConfigReader[Source.Kinesis.Buffer] =
224-
deriveReader[Source.Kinesis.Buffer]
225-
implicit val sinkConfigReader: ConfigReader[Sink] =
226-
deriveReader[Sink]
227-
implicit val sinkGoodConfigReader: ConfigReader[Sink.GoodSink] =
228-
deriveReader[Sink.GoodSink]
229-
implicit val sinkGoodStdoutConfigReader: ConfigReader[Sink.GoodSink.Stdout.type] =
230-
deriveReader[Sink.GoodSink.Stdout.type]
231-
implicit val sinkGoodESConfigReader: ConfigReader[Sink.GoodSink.Elasticsearch] =
232-
deriveReader[Sink.GoodSink.Elasticsearch]
233-
implicit val sinkGoodESClientConfigReader: ConfigReader[Sink.GoodSink.Elasticsearch.ESClient] =
234-
deriveReader[Sink.GoodSink.Elasticsearch.ESClient]
235-
implicit val sinkGoodESClusterConfigReader: ConfigReader[Sink.GoodSink.Elasticsearch.ESCluster] =
236-
deriveReader[Sink.GoodSink.Elasticsearch.ESCluster]
237-
implicit val sinkGoodESChunkConfigReader: ConfigReader[Sink.GoodSink.Elasticsearch.ESChunk] =
238-
deriveReader[Sink.GoodSink.Elasticsearch.ESChunk]
239-
implicit val sinkBadSinkConfigReader: ConfigReader[Sink.BadSink] =
240-
deriveReader[Sink.BadSink]
241-
implicit val sinkBadNoneConfigReader: ConfigReader[Sink.BadSink.None.type] =
242-
deriveReader[Sink.BadSink.None.type]
243-
implicit val sinkBadStderrConfigReader: ConfigReader[Sink.BadSink.Stderr.type] =
244-
deriveReader[Sink.BadSink.Stderr.type]
245-
implicit val sinkBadNsqConfigReader: ConfigReader[Sink.BadSink.Nsq] =
246-
deriveReader[Sink.BadSink.Nsq]
247-
implicit val monitoringConfigReader: ConfigReader[Monitoring] =
248-
deriveReader[Monitoring]
249-
implicit val snowplowMonitoringConfig: ConfigReader[Monitoring.SnowplowMonitoring] =
250-
deriveReader[Monitoring.SnowplowMonitoring]
251-
implicit val metricsConfigReader: ConfigReader[Monitoring.Metrics] =
252-
deriveReader[Monitoring.Metrics]
186+
final case class RawFailureReason(description: String) extends FailureReason
187+
final case class RawConfigReaderFailure(description: String, origin: Option[ConfigOrigin] = None)
188+
extends ConfigReaderFailure
189+
190+
case class implicits(
191+
regionConfigReader: ConfigReader[Region] with ReadsMissingKeys = new ConfigReader[Region]
192+
with ReadsMissingKeys {
193+
override def from(cur: ConfigCursor) =
194+
if (cur.isUndefined)
195+
Config.getRegion.toRight(
196+
ConfigReaderFailures(
197+
RawConfigReaderFailure(
198+
"Region can not be resolved, needs to be passed explicitly"
199+
)
200+
)
201+
)
202+
else
203+
cur.asString.flatMap { r =>
204+
val region = Region(r)
205+
checkRegion(region).leftMap(e => ConfigReaderFailures(RawConfigReaderFailure(e)))
206+
}
207+
}
208+
) {
209+
implicit val implRegionConfigReader: ConfigReader[Region] = regionConfigReader
210+
implicit val streamLoaderConfigReader: ConfigReader[StreamLoaderConfig] =
211+
deriveReader[StreamLoaderConfig]
212+
implicit val sourceConfigReader: ConfigReader[Source] =
213+
deriveReader[Source]
214+
implicit val sourceStdinConfigReader: ConfigReader[Source.Stdin.type] =
215+
deriveReader[Source.Stdin.type]
216+
implicit val sourceNsqConfigReader: ConfigReader[Source.Nsq] =
217+
deriveReader[Source.Nsq]
218+
implicit val sourceNsqBufferConfigReader: ConfigReader[Source.Nsq.Buffer] =
219+
deriveReader[Source.Nsq.Buffer]
220+
implicit val sourceKinesisConfigReader: ConfigReader[Source.Kinesis] =
221+
deriveReader[Source.Kinesis]
222+
implicit val sourceKinesisConfigBufferReader: ConfigReader[Source.Kinesis.Buffer] =
223+
deriveReader[Source.Kinesis.Buffer]
224+
implicit val sinkConfigReader: ConfigReader[Sink] =
225+
deriveReader[Sink]
226+
implicit val sinkGoodConfigReader: ConfigReader[Sink.GoodSink] =
227+
deriveReader[Sink.GoodSink]
228+
implicit val sinkGoodStdoutConfigReader: ConfigReader[Sink.GoodSink.Stdout.type] =
229+
deriveReader[Sink.GoodSink.Stdout.type]
230+
implicit val sinkGoodESConfigReader: ConfigReader[Sink.GoodSink.Elasticsearch] =
231+
deriveReader[Sink.GoodSink.Elasticsearch]
232+
implicit val sinkGoodESClientConfigReader: ConfigReader[Sink.GoodSink.Elasticsearch.ESClient] =
233+
deriveReader[Sink.GoodSink.Elasticsearch.ESClient]
234+
implicit val sinkGoodESClusterConfigReader: ConfigReader[
235+
Sink.GoodSink.Elasticsearch.ESCluster
236+
] =
237+
deriveReader[Sink.GoodSink.Elasticsearch.ESCluster]
238+
implicit val sinkGoodESAWSConfigReader: ConfigReader[Sink.GoodSink.Elasticsearch.ESAWS] =
239+
deriveReader[Sink.GoodSink.Elasticsearch.ESAWS]
240+
implicit val sinkGoodESChunkConfigReader: ConfigReader[Sink.GoodSink.Elasticsearch.ESChunk] =
241+
deriveReader[Sink.GoodSink.Elasticsearch.ESChunk]
242+
implicit val sinkBadSinkConfigReader: ConfigReader[Sink.BadSink] =
243+
deriveReader[Sink.BadSink]
244+
implicit val sinkBadKinesisConfigReader: ConfigReader[Sink.BadSink.Kinesis] =
245+
deriveReader[Sink.BadSink.Kinesis]
246+
implicit val sinkBadNoneConfigReader: ConfigReader[Sink.BadSink.None.type] =
247+
deriveReader[Sink.BadSink.None.type]
248+
implicit val sinkBadStderrConfigReader: ConfigReader[Sink.BadSink.Stderr.type] =
249+
deriveReader[Sink.BadSink.Stderr.type]
250+
implicit val sinkBadNsqConfigReader: ConfigReader[Sink.BadSink.Nsq] =
251+
deriveReader[Sink.BadSink.Nsq]
252+
implicit val monitoringConfigReader: ConfigReader[Monitoring] =
253+
deriveReader[Monitoring]
254+
implicit val snowplowMonitoringConfig: ConfigReader[Monitoring.SnowplowMonitoring] =
255+
deriveReader[Monitoring.SnowplowMonitoring]
256+
implicit val metricsConfigReader: ConfigReader[Monitoring.Metrics] =
257+
deriveReader[Monitoring.Metrics]
258+
}
253259

254260
val config = Opts
255261
.option[Path]("config", "Path to a HOCON configuration file")
@@ -263,7 +269,16 @@ object Config {
263269

264270
val command = Command("snowplow-stream-loader", generated.Settings.version, true)(config)
265271

266-
def parseConfig(arguments: Array[String]): Either[String, StreamLoaderConfig] =
272+
def parseConfig(arguments: Array[String]): Either[String, StreamLoaderConfig] = {
273+
val configImplicits = com.snowplowanalytics.stream.loader.Config.implicits()
274+
parseConfig(arguments, configImplicits.streamLoaderConfigReader)
275+
}
276+
277+
def parseConfig(
278+
arguments: Array[String],
279+
configReader: ConfigReader[StreamLoaderConfig]
280+
): Either[String, StreamLoaderConfig] = {
281+
implicit val implConfigReader: ConfigReader[StreamLoaderConfig] = configReader
267282
for {
268283
path <- command.parse(arguments).leftMap(_.toString)
269284
source = path.fold(ConfigSource.empty)(ConfigSource.file)
@@ -272,6 +287,7 @@ object Config {
272287
)
273288
parsed <- c.load[StreamLoaderConfig].leftMap(showFailures)
274289
} yield parsed
290+
}
275291

276292
/** Optionally give precedence to configs wrapped in a "esloader" block. To help avoid polluting config namespace */
277293
private def namespaced(configObjSource: ConfigObjectSource): ConfigObjectSource =
@@ -295,8 +311,14 @@ object Config {
295311
failureStrings.mkString("\n")
296312
}
297313

298-
private def getRegion: Option[String] =
299-
Either.catchNonFatal((new DefaultAwsRegionProviderChain).getRegion).toOption
314+
private def getRegion: Option[Region] =
315+
Either.catchNonFatal((new DefaultAwsRegionProviderChain).getRegion).toOption.map(Region)
316+
317+
private def checkRegion(region: Region): Either[String, Region] = {
318+
val allRegions = Regions.values().toList.map(_.getName)
319+
if (allRegions.contains(region.name)) region.asRight
320+
else s"Region ${region.name} is unknown, choose from [${allRegions.mkString(", ")}]".asLeft
321+
}
300322

301323
// Used as an option prefix when reading system properties.
302324
val Namespace = "snowplow"

‎core/src/main/scala/com.snowplowanalytics.stream/loader/executors/KinesisSourceExecutor.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -87,12 +87,12 @@ class KinesisSourceExecutor[A, B](
8787
+ kcc.CONNECTOR_DESTINATION + ","
8888
+ KinesisConnectorConfiguration.KINESIS_CONNECTOR_USER_AGENT
8989
)
90+
.withRegionName(kcc.REGION_NAME)
9091
.condWith(kinesis.customEndpoint.isDefined, _.withKinesisEndpoint(kcc.KINESIS_ENDPOINT))
9192
.condWith(
9293
kinesis.dynamodbCustomEndpoint.isDefined,
9394
_.withDynamoDBEndpoint(kcc.DYNAMODB_ENDPOINT)
9495
)
95-
.condWith(kinesis.region.isDefined, _.withRegionName(kcc.REGION_NAME))
9696

9797
timestamp
9898
.filter(_ => initialPosition == "AT_TIMESTAMP")
@@ -120,7 +120,7 @@ class KinesisSourceExecutor[A, B](
120120
props.setProperty(KinesisConnectorConfiguration.PROP_DYNAMODB_ENDPOINT, _)
121121
)
122122
// So that the region of the DynamoDB table is correct
123-
kinesis.region.foreach(props.setProperty(KinesisConnectorConfiguration.PROP_REGION_NAME, _))
123+
props.setProperty(KinesisConnectorConfiguration.PROP_REGION_NAME, kinesis.region.name)
124124
props.setProperty(KinesisConnectorConfiguration.PROP_APP_NAME, kinesis.appName.trim)
125125
props.setProperty(
126126
KinesisConnectorConfiguration.PROP_INITIAL_POSITION_IN_STREAM,

‎core/src/main/scala/com.snowplowanalytics.stream/loader/sinks/KinesisSink.scala

+4-7
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,10 @@ class KinesisSink(conf: KinesisSinkConfig) extends ISink {
5454
val client = AmazonKinesisClientBuilder
5555
.standard()
5656
.withCredentials(new DefaultAWSCredentialsProviderChain())
57-
.optWith[String](conf.region, _.withRegion)
58-
.optWith[(String, String)](
59-
for {
60-
e <- conf.customEndpoint
61-
r <- conf.region
62-
} yield (e, r),
63-
b => { case (e, r) => b.withEndpointConfiguration(new EndpointConfiguration(e, r)) }
57+
.withRegion(conf.region.name)
58+
.optWith[String](
59+
conf.customEndpoint,
60+
b => e => b.withEndpointConfiguration(new EndpointConfiguration(e, conf.region.name))
6461
)
6562
.build()
6663

‎core/src/test/resources/config.test1.hocon

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"input": {
3+
"type": "kinesis"
4+
"streamName": "test-kinesis-stream"
5+
"appName": "test-app-name"
6+
"initialPosition": "LATEST"
7+
"region": "ca-central-1"
8+
"maxRecords": 2000
9+
"buffer": {
10+
"byteLimit": 201
11+
"recordLimit": 202
12+
"timeLimit": 203
13+
}
14+
}
15+
"output": {
16+
"type": "elasticsearch"
17+
"good": {
18+
"client": {
19+
"endpoint": "localhost"
20+
"maxTimeout": 205
21+
"maxRetries": 7
22+
"port": 9220
23+
"ssl": true
24+
}
25+
"chunk": {
26+
"byteLimit": 206
27+
"recordLimit": 207
28+
}
29+
"aws": {
30+
"signing": true
31+
"region": "ca-central-1"
32+
}
33+
"cluster": {
34+
"index": "testindex"
35+
}
36+
}
37+
"bad" {
38+
"type": "kinesis"
39+
"streamName": "test-kinesis-bad-stream"
40+
"region": "ca-central-1"
41+
}
42+
}
43+
"purpose": "BAD_ROWS"
44+
"monitoring": {
45+
"metrics": {
46+
"cloudWatch": false
47+
}
48+
}
49+
}

0 commit comments

Comments
 (0)