Path normalization via new NodeTransformer interface (#413)

Co-authored-by: Sam <[email protected]>

hoplite-aws/src/main/kotlin/com/sksamuel/hoplite/aws/ParameterStorePathPropertySource.kt

@@ -10,7 +10,6 @@ import com.sksamuel.hoplite.PropertySource
 import com.sksamuel.hoplite.PropertySourceContext
 import com.sksamuel.hoplite.decoder.toValidated
 import com.sksamuel.hoplite.parsers.toNode
-import java.util.Properties
 
 /**
  * Provides all keys under a prefix path as config values.
@@ -47,12 +46,9 @@ class ParameterStorePathPropertySource(
 
   override fun node(context: PropertySourceContext): ConfigResult<Node> {
     return fetchParameterStoreValues().map { params ->
-      val props = Properties()
-      params.forEach {
-        val name = if (stripPath) it.name.removePrefix(prefix) else it.name
-        props[name.removePrefix("/")] = it.value
+      params.associate { it.name to it.value }.toNode("aws_parameter_store at $prefix", "/") {
+        (if (stripPath) it.removePrefix(prefix) else it).removePrefix("/")
       }
-      props.toNode("aws_parameter_store at $prefix", "/")
     }.toValidated {
       ConfigFailure.PropertySourceFailure("Could not fetch data from AWS parameter store: ${it.message}", it)
     }

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigFailure.kt

@@ -1,7 +1,6 @@
 package com.sksamuel.hoplite
 
 import com.sksamuel.hoplite.decoder.Decoder
-import com.sksamuel.hoplite.decoder.DotPath
 import com.sksamuel.hoplite.fp.NonEmptyList
 import com.sksamuel.hoplite.internal.OverridePath
 import com.sksamuel.hoplite.parsers.Parser
@@ -30,9 +29,9 @@ sealed interface ConfigFailure {
    */
   fun description(): String
 
-  data class UnusedPath(val path: DotPath, val pos: Pos) : ConfigFailure {
+  data class UnusedPath(val decodedPath: DecodedPath) : ConfigFailure {
     override fun description(): String {
-      return "Config value '${path.flatten()}' at ${pos.loc()} was unused"
+      return "Config value '${decodedPath.sourceKey ?: decodedPath.path.flatten()}' at ${decodedPath.pos.loc()} was unused"
     }
   }
 

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoader.kt

@@ -9,6 +9,7 @@ import com.sksamuel.hoplite.fp.getOrElse
 import com.sksamuel.hoplite.internal.CascadeMode
 import com.sksamuel.hoplite.internal.ConfigParser
 import com.sksamuel.hoplite.internal.DecodeMode
+import com.sksamuel.hoplite.transformer.NodeTransformer
 import com.sksamuel.hoplite.parsers.ParserRegistry
 import com.sksamuel.hoplite.preprocessor.Preprocessor
 import com.sksamuel.hoplite.report.Print
@@ -27,6 +28,7 @@ class ConfigLoader(
   val propertySources: List<PropertySource>,
   val parserRegistry: ParserRegistry,
   val preprocessors: List<Preprocessor>,
+  val nodeTransformers: List<NodeTransformer>,
   val paramMappers: List<ParameterMapper>,
   val onFailure: List<(Throwable) -> Unit> = emptyList(),
   val decodeMode: DecodeMode = DecodeMode.Lenient,
@@ -195,6 +197,7 @@ class ConfigLoader(
       cascadeMode = cascadeMode,
       preprocessors = preprocessors,
       preprocessingIterations = preprocessingIterations,
+      nodeTransformers = nodeTransformers,
       prefix = prefix,
       resolvers = resolvers,
       decoderRegistry = decoderRegistry,
@@ -251,6 +254,7 @@ class ConfigLoader(
       cascadeMode = cascadeMode,
       preprocessors = preprocessors,
       preprocessingIterations = preprocessingIterations,
+      nodeTransformers = nodeTransformers,
       prefix = null,
       resolvers = resolvers,
       decoderRegistry = decoderRegistry,

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ConfigLoaderBuilder.kt

@@ -5,6 +5,7 @@ import com.sksamuel.hoplite.decoder.DefaultDecoderRegistry
 import com.sksamuel.hoplite.env.Environment
 import com.sksamuel.hoplite.internal.CascadeMode
 import com.sksamuel.hoplite.internal.DecodeMode
+import com.sksamuel.hoplite.transformer.NodeTransformer
 import com.sksamuel.hoplite.parsers.DefaultParserRegistry
 import com.sksamuel.hoplite.parsers.Parser
 import com.sksamuel.hoplite.preprocessor.EnvOrSystemPropertyPreprocessor
@@ -30,6 +31,7 @@ import com.sksamuel.hoplite.sources.EnvironmentVariableOverridePropertySource
 import com.sksamuel.hoplite.sources.SystemPropertiesPropertySource
 import com.sksamuel.hoplite.sources.UserSettingsPropertySource
 import com.sksamuel.hoplite.sources.XdgConfigPropertySource
+import com.sksamuel.hoplite.transformer.PathNormalizer
 import java.util.ServiceLoader
 
 class ConfigLoaderBuilder private constructor() {
@@ -51,6 +53,7 @@ class ConfigLoaderBuilder private constructor() {
 
   private val propertySources = mutableListOf<PropertySource>()
   private val preprocessors = mutableListOf<Preprocessor>()
+  private val nodeTransformers = mutableListOf<NodeTransformer>()
   private val resolvers = mutableListOf<Resolver>()
   private val paramMappers = mutableListOf<ParameterMapper>()
   private val parsers = mutableMapOf<String, Parser>()
@@ -70,8 +73,8 @@ class ConfigLoaderBuilder private constructor() {
     /**
      * Returns a [ConfigLoaderBuilder] with all defaults applied.
      *
-     * This means that the default [Decoder]s, [Preprocessor]s, [ParameterMapper]s, [PropertySource]s,
-     * and [Parser]s are all registered.
+     * This means that the default [Decoder]s, [Preprocessor]s, [NodeTransformer]s, [ParameterMapper]s,
+     * [PropertySource]s, and [Parser]s are all registered.
      *
      * If you wish to avoid adding defaults, for example to avoid certain decoders or sources, then
      * use [empty] to obtain an empty ConfigLoaderBuilder and call the various addDefault methods manually.
@@ -80,6 +83,7 @@ class ConfigLoaderBuilder private constructor() {
       return empty()
         .addDefaultDecoders()
         .addDefaultPreprocessors()
+        .addDefaultNodeTransformers()
         .addDefaultParamMappers()
         .addDefaultPropertySources()
         .addDefaultParsers()
@@ -88,7 +92,7 @@ class ConfigLoaderBuilder private constructor() {
     /**
      * Returns a [ConfigLoaderBuilder] with all defaults applied, using resolvers in place of preprocessors.
      *
-     * This means that the default [Decoder]s, [Resolver]s, [ParameterMapper]s, [PropertySource]s,
+     * This means that the default [Decoder]s, [Resolver]s, [NodeTransformer]s, [ParameterMapper]s, [PropertySource]s,
      * and [Parser]s are all registered.
      *
      * If you wish to avoid adding defaults, for example to avoid certain decoders or sources, then
@@ -102,6 +106,7 @@ class ConfigLoaderBuilder private constructor() {
       return empty()
         .addDefaultDecoders()
         .addDefaultResolvers()
+        .addDefaultNodeTransformers()
         .addDefaultParamMappers()
         .addDefaultPropertySources()
         .addDefaultParsers()
@@ -205,6 +210,16 @@ class ConfigLoaderBuilder private constructor() {
 
   fun addDefaultPreprocessors() = addPreprocessors(defaultPreprocessors())
 
+  fun addNodeTransformer(nodeTransformer: NodeTransformer): ConfigLoaderBuilder = apply {
+    this.nodeTransformers.add(nodeTransformer)
+  }
+
+  fun addNodeTransformers(nodeTransformers: Iterable<NodeTransformer>): ConfigLoaderBuilder = apply {
+    this.nodeTransformers.addAll(nodeTransformers)
+  }
+
+  fun addDefaultNodeTransformers() = addNodeTransformers(defaultNodeTransformers())
+
   fun addParser(ext: String, parser: Parser) = addFileExtensionMapping(ext, parser)
   fun addParsers(map: Map<String, Parser>) = addFileExtensionMappings(map)
 
@@ -249,6 +264,7 @@ class ConfigLoaderBuilder private constructor() {
     return addDefaultDecoders()
       .addDefaultParsers()
       .addDefaultPreprocessors()
+      .addDefaultNodeTransformers()
       .addDefaultParamMappers()
       .addDefaultPropertySources()
   }
@@ -372,6 +388,7 @@ class ConfigLoaderBuilder private constructor() {
       propertySources = propertySources.toList(),
       parserRegistry = DefaultParserRegistry(parsers),
       preprocessors = preprocessors.toList(),
+      nodeTransformers = nodeTransformers.toList(),
       paramMappers = paramMappers.toList(),
       onFailure = failureCallbacks.toList(),
       resolvers = resolvers,
@@ -407,6 +424,10 @@ fun defaultPreprocessors(): List<Preprocessor> = listOf(
   LookupPreprocessor,
 )
 
+fun defaultNodeTransformers(): List<NodeTransformer> = listOf(
+  PathNormalizer,
+)
+
 fun defaultResolvers(): List<Resolver> = listOf(
   EnvVarContextResolver,
   SystemPropertyContextResolver,
@@ -419,8 +440,7 @@ fun defaultResolvers(): List<Resolver> = listOf(
 
 fun defaultParamMappers(): List<ParameterMapper> = listOf(
   DefaultParamMapper,
-  SnakeCaseParamMapper,
-  KebabCaseParamMapper,
+  LowercaseParamMapper,
   AliasAnnotationParamMapper,
 )
 

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/DecoderContext.kt

@@ -75,3 +75,9 @@ data class DecoderConfig(
   val flattenArraysToString: Boolean,
   val resolveTypesCaseInsensitive: Boolean,
 )
+
+data class DecodedPath(
+  val path: DotPath,
+  val sourceKey: String?,
+  val pos: Pos,
+)

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/ParameterMapper.kt

@@ -34,6 +34,11 @@ object DefaultParamMapper : ParameterMapper {
     setOfNotNull(param.name)
 }
 
+object LowercaseParamMapper : ParameterMapper {
+  override fun map(param: KParameter, constructor: KFunction<Any>, kclass: KClass<*>): Set<String> =
+    setOfNotNull(param.name?.lowercase())
+}
+
 /**
  * Disabled by default so that common ENVVAR PARAMS don't override your lower case
  * names unexpectedly.
@@ -57,6 +62,8 @@ object AliasAnnotationParamMapper : ParameterMapper {
  * the snake case equivalent.
  *
  * For example, camelCasePilsen will become snake_case_pilsen.
+ *
+ * When using the [PathNormalizer] (which is enabled by default), this mapper is unnecessary.
  */
 object SnakeCaseParamMapper : ParameterMapper {
 
@@ -81,6 +88,8 @@ object SnakeCaseParamMapper : ParameterMapper {
  * the kebab case equivalent.
  *
  * For example, camelCasePilsen will become kebab-case-pilsen.
+ *
+ * When using the [PathNormalizer] (which is enabled by default), this mapper is unnecessary.
  */
 object KebabCaseParamMapper : ParameterMapper {
 

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/decoder/AbstractUnnormalizedKeysDecoder.kt

@@ -0,0 +1,33 @@
+package com.sksamuel.hoplite.decoder
+
+import com.sksamuel.hoplite.ConfigResult
+import com.sksamuel.hoplite.DecoderContext
+import com.sksamuel.hoplite.MapNode
+import com.sksamuel.hoplite.Node
+import com.sksamuel.hoplite.transform
+import kotlin.reflect.KType
+
+/**
+ * A decoder which decodes based on unnormalized keys.
+ *
+ * This is useful for decoders that need to know the original key names.
+ *
+ * It restores the original key names from the node source key.
+ */
+abstract class AbstractUnnormalizedKeysDecoder<T> : NullHandlingDecoder<T> {
+  override fun safeDecode(node: Node, type: KType, context: DecoderContext): ConfigResult<T> {
+    val unnormalizedNode = node.transform {
+      val sourceKey = it.sourceKey
+      when (it) {
+        is MapNode -> it.copy(map = it.map.mapKeys { (k, v) ->
+          (v.sourceKey ?: k).removePrefix("$sourceKey.")
+        })
+        else -> it
+      }
+    }
+
+    return safeDecodeUnnormalized(unnormalizedNode, type, context)
+  }
+
+  abstract fun safeDecodeUnnormalized(node: Node, type: KType, context: DecoderContext): ConfigResult<T>
+}

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/decoder/MapDecoder.kt

@@ -15,15 +15,13 @@ import kotlin.reflect.full.isSubtypeOf
 import kotlin.reflect.full.starProjectedType
 import kotlin.reflect.full.withNullability
 
-class MapDecoder : NullHandlingDecoder<Map<*, *>> {
+class MapDecoder : AbstractUnnormalizedKeysDecoder<Map<*, *>>() {
 
   override fun supports(type: KType): Boolean =
     type.isSubtypeOf(Map::class.starProjectedType) ||
       type.isSubtypeOf(Map::class.starProjectedType.withNullability(true))
 
-  override fun safeDecode(node: Node,
-                          type: KType,
-                          context: DecoderContext): ConfigResult<Map<*, *>> {
+  override fun safeDecodeUnnormalized(node: Node, type: KType, context: DecoderContext): ConfigResult<Map<*, *>> {
     require(type.arguments.size == 2)
 
     val kType = type.arguments[0].type!!

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/ConfigParser.kt

@@ -14,6 +14,7 @@ import com.sksamuel.hoplite.env.Environment
 import com.sksamuel.hoplite.fp.flatMap
 import com.sksamuel.hoplite.fp.invalid
 import com.sksamuel.hoplite.fp.valid
+import com.sksamuel.hoplite.transformer.NodeTransformer
 import com.sksamuel.hoplite.parsers.ParserRegistry
 import com.sksamuel.hoplite.preprocessor.Preprocessor
 import com.sksamuel.hoplite.report.Print
@@ -23,34 +24,36 @@ import com.sksamuel.hoplite.resolver.Resolving
 import com.sksamuel.hoplite.resolver.context.ContextResolverMode
 import com.sksamuel.hoplite.secrets.Obfuscator
 import com.sksamuel.hoplite.secrets.SecretsPolicy
+import com.sksamuel.hoplite.transformer.PathNormalizer
 import kotlin.reflect.KClass
 
 class ConfigParser(
-   classpathResourceLoader: ClasspathResourceLoader,
-   parserRegistry: ParserRegistry,
-   allowEmptyTree: Boolean,
-   allowNullOverride: Boolean,
-   cascadeMode: CascadeMode,
-   preprocessors: List<Preprocessor>,
-   preprocessingIterations: Int,
-   private val prefix: String?,
-   private val resolvers: List<Resolver>,
-   private val decoderRegistry: DecoderRegistry,
-   private val paramMappers: List<ParameterMapper>,
-   private val flattenArraysToString: Boolean,
-   private val resolveTypesCaseInsensitive: Boolean,
-   private val allowUnresolvedSubstitutions: Boolean,
-   private val secretsPolicy: SecretsPolicy?,
-   private val decodeMode: DecodeMode,
-   private val useReport: Boolean,
-   private val obfuscator: Obfuscator,
-   private val reportPrintFn: Print,
-   private val environment: Environment?,
-   private val sealedTypeDiscriminatorField: String?,
-   private val contextResolverMode: ContextResolverMode,
+  classpathResourceLoader: ClasspathResourceLoader,
+  parserRegistry: ParserRegistry,
+  allowEmptyTree: Boolean,
+  allowNullOverride: Boolean,
+  cascadeMode: CascadeMode,
+  preprocessors: List<Preprocessor>,
+  preprocessingIterations: Int,
+  private val nodeTransformers: List<NodeTransformer>,
+  private val prefix: String?,
+  private val resolvers: List<Resolver>,
+  private val decoderRegistry: DecoderRegistry,
+  private val paramMappers: List<ParameterMapper>,
+  private val flattenArraysToString: Boolean,
+  private val resolveTypesCaseInsensitive: Boolean,
+  private val allowUnresolvedSubstitutions: Boolean,
+  private val secretsPolicy: SecretsPolicy?,
+  private val decodeMode: DecodeMode,
+  private val useReport: Boolean,
+  private val obfuscator: Obfuscator,
+  private val reportPrintFn: Print,
+  private val environment: Environment?,
+  private val sealedTypeDiscriminatorField: String?,
+  private val contextResolverMode: ContextResolverMode,
 ) {
 
-  private val loader = PropertySourceLoader(classpathResourceLoader, parserRegistry, allowEmptyTree)
+  private val loader = PropertySourceLoader(nodeTransformers, sealedTypeDiscriminatorField, classpathResourceLoader, parserRegistry, allowEmptyTree)
   private val cascader = Cascader(cascadeMode, allowEmptyTree, allowNullOverride)
   private val preprocessing = Preprocessing(preprocessors, preprocessingIterations)
   private val decoding = Decoding(decoderRegistry, secretsPolicy)
@@ -82,8 +85,7 @@ class ConfigParser(
       cascader.cascade(nodes).flatMap { node ->
         val context = context(node)
         preprocessing.preprocess(node, context).flatMap { preprocessed ->
-          check(preprocessed.let { if (prefix == null) it else it.atPath(prefix) }).flatMap {
-
+          check(preprocessed.prefixedNode()).flatMap {
             val decoded = decoding.decode(kclass, it, decodeMode, context)
             val state = createDecodingState(it, context, secretsPolicy)
 
@@ -122,4 +124,10 @@ class ConfigParser(
     else
       UnresolvedSubstitutionChecker.process(node)
   }
+
+  private fun Node.prefixedNode() = when {
+    prefix == null -> this
+    nodeTransformers.contains(PathNormalizer) -> atPath(PathNormalizer.normalizePathElement(prefix))
+    else -> atPath(prefix)
+  }
 }

hoplite-core/src/main/kotlin/com/sksamuel/hoplite/internal/DecodeModeValidator.kt

@@ -20,7 +20,7 @@ class DecodeModeValidator(private val mode: DecodeMode) {
 
   private fun ensureAllUsed(result: DecodingState): ConfigResult<DecodingState> {
     return if (result.unused.isEmpty()) result.valid() else {
-      val errors = NonEmptyList.unsafe(result.unused.map { ConfigFailure.UnusedPath(it.first, it.second) })
+      val errors = NonEmptyList.unsafe(result.unused.map { ConfigFailure.UnusedPath(it) })
       ConfigFailure.MultipleFailures(errors).invalid()
     }
   }