From d1df046c870335327987df7173d64ba5397e92ee Mon Sep 17 00:00:00 2001 From: Lorenzo Delgado Date: Thu, 3 Nov 2022 17:58:48 +0100 Subject: [PATCH] feat(wakunode2): support configuration via environment variables --- apps/wakunode2/config.nim | 15 ++- apps/wakunode2/wakunode2.nim | 2 +- tests/all_tests_v2.nim | 7 +- tests/v2/test_confutils_envvar.nim | 81 +++++++++++++++ tests/v2/test_envvar_serialization.nim | 20 ++++ waku/common/confutils/envvar/defs.nim | 24 +++++ waku/common/confutils/envvar/std/net.nim | 28 +++++ waku/common/envvar_serialization.nim | 63 ++++++++++++ waku/common/envvar_serialization/reader.nim | 95 +++++++++++++++++ waku/common/envvar_serialization/utils.nim | 108 ++++++++++++++++++++ waku/common/envvar_serialization/writer.nim | 54 ++++++++++ 11 files changed, 494 insertions(+), 3 deletions(-) create mode 100644 tests/v2/test_confutils_envvar.nim create mode 100644 tests/v2/test_envvar_serialization.nim create mode 100644 waku/common/confutils/envvar/defs.nim create mode 100644 waku/common/confutils/envvar/std/net.nim create mode 100644 waku/common/envvar_serialization.nim create mode 100644 waku/common/envvar_serialization/reader.nim create mode 100644 waku/common/envvar_serialization/utils.nim create mode 100644 waku/common/envvar_serialization/writer.nim diff --git a/apps/wakunode2/config.nim b/apps/wakunode2/config.nim index 4ff2566dde..af5f29a857 100644 --- a/apps/wakunode2/config.nim +++ b/apps/wakunode2/config.nim @@ -12,10 +12,15 @@ import libp2p/crypto/crypto, libp2p/crypto/secp, nimcrypto/utils +import + ../../waku/common/confutils/envvar/defs as confEnvvarDefs, + ../../waku/common/confutils/envvar/std/net as confEnvvarNet export confTomlDefs, - confTomlNet + confTomlNet, + confEnvvarDefs, + confEnvvarNet type ConfResult*[T] = Result[T, string] @@ -506,6 +511,12 @@ proc readValue*(r: var TomlReader, value: var crypto.PrivateKey) {.raises: [Seri except CatchableError: raise newException(SerializationError, getCurrentExceptionMsg()) +proc readValue*(r: var EnvvarReader, value: var crypto.PrivateKey) {.raises: [SerializationError].} = + try: + value = parseCmdArg(crypto.PrivateKey, r.readValue(string)) + except CatchableError: + raise newException(SerializationError, getCurrentExceptionMsg()) + {.push warning[ProveInit]: off.} @@ -514,6 +525,8 @@ proc load*(T: type WakuNodeConf, version=""): ConfResult[T] = let conf = WakuNodeConf.load( version=version, secondarySources = proc (conf: WakuNodeConf, sources: auto) = + sources.addConfigFile(Envvar, InputFile("wakunode2")) + if conf.configFile.isSome(): sources.addConfigFile(Toml, conf.configFile.get()) ) diff --git a/apps/wakunode2/wakunode2.nim b/apps/wakunode2/wakunode2.nim index ef310a2a6d..239e7ff044 100644 --- a/apps/wakunode2/wakunode2.nim +++ b/apps/wakunode2/wakunode2.nim @@ -557,7 +557,7 @@ when isMainModule: if conf.logLevel != LogLevel.NONE: setLogLevel(conf.logLevel) - + ############## # Node setup # ############## diff --git a/tests/all_tests_v2.nim b/tests/all_tests_v2.nim index a1fdc0d2a7..649c0092a8 100644 --- a/tests/all_tests_v2.nim +++ b/tests/all_tests_v2.nim @@ -1,3 +1,9 @@ +import + # Waku common tests + ./v2/test_envvar_serialization, + ./v2/test_confutils_envvar, + ./v2/test_sqlite_migrations + import # Waku v2 tests ./v2/test_wakunode, @@ -36,7 +42,6 @@ import ./v2/test_waku_bridge, ./v2/test_peer_storage, ./v2/test_waku_keepalive, - ./v2/test_sqlite_migrations, ./v2/test_namespacing_utils, ./v2/test_waku_dnsdisc, ./v2/test_waku_discv5, diff --git a/tests/v2/test_confutils_envvar.nim b/tests/v2/test_confutils_envvar.nim new file mode 100644 index 0000000000..551f9f098f --- /dev/null +++ b/tests/v2/test_confutils_envvar.nim @@ -0,0 +1,81 @@ +{.used.} + +import + std/[os, options], + stew/results, + stew/shims/net as stewNet, + testutils/unittests, + confutils, + confutils/defs, + confutils/std/net +import + ../../waku/common/confutils/envvar/defs as confEnvvarDefs, + ../../waku/common/confutils/envvar/std/net as confEnvvarNet + + +type ConfResult[T] = Result[T, string] + +type TestConf = object + configFile* {. + desc: "Configuration file path" + name: "config-file" }: Option[InputFile] + + testFile* {. + desc: "Configuration test file path" + name: "test-file" }: Option[InputFile] + + listenAddress* {. + defaultValue: ValidIpAddress.init("127.0.0.1"), + desc: "Listening address", + name: "listen-address"}: ValidIpAddress + + tcpPort* {. + desc: "TCP listening port", + defaultValue: 60000, + name: "tcp-port" }: Port + + +{.push warning[ProveInit]: off.} + +proc load*(T: type TestConf, prefix: string): ConfResult[T] = + try: + let conf = TestConf.load( + secondarySources = proc (conf: TestConf, sources: auto) = + sources.addConfigFile(Envvar, InputFile(prefix)) + ) + ok(conf) + except CatchableError: + err(getCurrentExceptionMsg()) + +{.pop.} + + +suite "nim-confutils - envvar": + test "load configuration from environment variables": + ## Given + let prefix = "test-prefix" + + let + listenAddress = "1.1.1.1" + tcpPort = "8080" + configFile = "/tmp/test.conf" + + ## When + os.putEnv("TEST_PREFIX_CONFIG_FILE", configFile) + os.putEnv("TEST_PREFIX_LISTEN_ADDRESS", listenAddress) + os.putEnv("TEST_PREFIX_TCP_PORT", tcpPort) + + let confLoadRes = TestConf.load(prefix) + + ## Then + check confLoadRes.isOk() + + let conf = confLoadRes.get() + check: + conf.listenAddress == ValidIpAddress.init(listenAddress) + conf.tcpPort == Port(8080) + + conf.configFile.isSome() + conf.configFile.get().string == configFile + + conf.testFile.isNone() \ No newline at end of file diff --git a/tests/v2/test_envvar_serialization.nim b/tests/v2/test_envvar_serialization.nim new file mode 100644 index 0000000000..9700a14f86 --- /dev/null +++ b/tests/v2/test_envvar_serialization.nim @@ -0,0 +1,20 @@ +{.used.} + +import + testutils/unittests +import + ../../waku/common/envvar_serialization/utils + + +suite "nim-envvar-serialization - utils": + test "construct env var key": + ## Given + let prefix = "some-prefix" + let name = @["db-url"] + + ## When + let key = constructKey(prefix, name) + + ## Then + check: + key == "SOME_PREFIX_DB_URL" \ No newline at end of file diff --git a/waku/common/confutils/envvar/defs.nim b/waku/common/confutils/envvar/defs.nim new file mode 100644 index 0000000000..75bf06e3d8 --- /dev/null +++ b/waku/common/confutils/envvar/defs.nim @@ -0,0 +1,24 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + + +import + confutils/defs as confutilsDefs +import + ../../envvar_serialization + +export + envvar_serialization, confutilsDefs + + +template readConfutilsType(T: type) = + template readValue*(r: var EnvvarReader, value: var T) = + value = T r.readValue(string) + +readConfutilsType InputFile +readConfutilsType InputDir +readConfutilsType OutPath +readConfutilsType OutDir +readConfutilsType OutFile diff --git a/waku/common/confutils/envvar/std/net.nim b/waku/common/confutils/envvar/std/net.nim new file mode 100644 index 0000000000..2b58ee8dc0 --- /dev/null +++ b/waku/common/confutils/envvar/std/net.nim @@ -0,0 +1,28 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + + +import + std/strutils, + stew/shims/net +import + ../../../envvar_serialization + +export + net, + envvar_serialization + + +proc readValue*(r: var EnvvarReader, value: var ValidIpAddress) {.raises: [SerializationError].} = + try: + value = ValidIpAddress.init(r.readValue(string)) + except ValueError: + raise newException(EnvvarError, "Invalid IP address") + +proc readValue*(r: var EnvvarReader, value: var Port) {.raises: [SerializationError, ValueError].} = + try: + value = parseUInt(r.readValue(string)).Port + except ValueError: + raise newException(EnvvarError, "Invalid Port") diff --git a/waku/common/envvar_serialization.nim b/waku/common/envvar_serialization.nim new file mode 100644 index 0000000000..0d20af8eaf --- /dev/null +++ b/waku/common/envvar_serialization.nim @@ -0,0 +1,63 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + + +import + stew/shims/macros, + serialization +import + ./envvar_serialization/reader, + ./envvar_serialization/writer + +export + serialization, + reader, + writer + + +serializationFormat Envvar + +Envvar.setReader EnvvarReader +Envvar.setWriter EnvvarWriter, PreferredOutput = void + + +template supports*(_: type Envvar, T: type): bool = + # The Envvar format should support every type + true + +template decode*(_: type Envvar, + prefix: string, + RecordType: distinct type, + params: varargs[untyped]): auto = + mixin init, ReaderType + + {.noSideEffect.}: + var reader = unpackArgs(init, [EnvvarReader, prefix, params]) + reader.readValue(RecordType) + +template encode*(_: type Envvar, + prefix: string, + value: auto, + params: varargs[untyped]) = + mixin init, WriterType, writeValue + + {.noSideEffect.}: + var writer = unpackArgs(init, [EnvvarWriter, prefix, params]) + writeValue writer, value + +template loadFile*(_: type Envvar, + prefix: string, + RecordType: distinct type, + params: varargs[untyped]): auto = + mixin init, ReaderType, readValue + + var reader = unpackArgs(init, [EnvvarReader, prefix, params]) + reader.readValue(RecordType) + +template saveFile*(_: type Envvar, prefix: string, value: auto, params: varargs[untyped]) = + mixin init, WriterType, writeValue + + var writer = unpackArgs(init, [EnvvarWriter, prefix, params]) + writer.writeValue(value) diff --git a/waku/common/envvar_serialization/reader.nim b/waku/common/envvar_serialization/reader.nim new file mode 100644 index 0000000000..837935534c --- /dev/null +++ b/waku/common/envvar_serialization/reader.nim @@ -0,0 +1,95 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + + +import + std/[tables, typetraits, options, os], + serialization/object_serialization, + serialization/errors +import + ./utils + + +type + EnvvarReader* = object + prefix: string + key: seq[string] + + EnvvarError* = object of SerializationError + + EnvvarReaderError* = object of EnvvarError + + GenericEnvvarReaderError* = object of EnvvarReaderError + deserializedField*: string + innerException*: ref CatchableError + + +proc handleReadException*(r: EnvvarReader, + Record: type, + fieldName: string, + field: auto, + err: ref CatchableError) {.raises: [GenericEnvvarReaderError].} = + var ex = new GenericEnvvarReaderError + ex.deserializedField = fieldName + ex.innerException = err + raise ex + +proc init*(T: type EnvvarReader, prefix: string): T = + result.prefix = prefix + +proc readValue*[T](r: var EnvvarReader, value: var T) {.raises: [ValueError, SerializationError].} = + mixin readValue + + when T is string: + let key = constructKey(r.prefix, r.key) + value = os.getEnv(key) + + elif T is (SomePrimitives or range): + let key = constructKey(r.prefix, r.key) + getValue(key, value) + + elif T is Option: + template getUnderlyingType[T](_: Option[T]): untyped = T + let key = constructKey(r.prefix, r.key) + if os.existsEnv(key): + type uType = getUnderlyingType(value) + when uType is string: + value = some(os.getEnv(key)) + else: + value = some(r.readValue(uType)) + + elif T is (seq or array): + when uTypeIsPrimitives(T): + let key = constructKey(r.prefix, r.key) + getValue(key, value) + + else: + let key = r.key[^1] + for i in 0.. 0: + let fields = T.fieldReadersTable(EnvvarReader) + var expectedFieldPos = 0 + r.key.add "" + value.enumInstanceSerializedFields(fieldName, field): + when T is tuple: + r.key[^1] = $expectedFieldPos + var reader = fields[][expectedFieldPos].reader + expectedFieldPos += 1 + else: + r.key[^1] = fieldName + var reader = findFieldReader(fields[], fieldName, expectedFieldPos) + + if reader != nil: + reader(value, r) + discard r.key.pop() + + else: + const typeName = typetraits.name(T) + {.fatal: "Failed to convert from Envvar an unsupported type: " & typeName.} diff --git a/waku/common/envvar_serialization/utils.nim b/waku/common/envvar_serialization/utils.nim new file mode 100644 index 0000000000..fee6ece4fb --- /dev/null +++ b/waku/common/envvar_serialization/utils.nim @@ -0,0 +1,108 @@ +when (NimMajor, NimMinor) < (1, 4): + {.push raises: [Defect].} +else: + {.push raises: [].} + + +import + std/[os, strutils], + stew/byteutils, + stew/ranges/ptr_arith + + +type + SomePrimitives* = SomeInteger | enum | bool | SomeFloat | char + +proc setValue*[T: SomePrimitives](key: string, val: openArray[T]) = + os.putEnv(key, byteutils.toHex(makeOpenArray(val[0].unsafeAddr, byte, val.len*sizeof(T)))) + +proc setValue*(key: string, val: SomePrimitives) = + os.putEnv(key, byteutils.toHex(makeOpenArray(val.unsafeAddr, byte, sizeof(val)))) + +proc decodePaddedHex(hex: string, res: ptr UncheckedArray[byte], outputLen: int) {.raises: [ValueError].} = + # make it an even length + let + inputLen = hex.len and not 0x01 + numHex = inputLen div 2 + maxLen = min(outputLen, numHex) + + var + offI = hex.len - maxLen * 2 + offO = outputLen - maxLen + + for i in 0 ..< maxLen: + res[i + offO] = hex[2*i + offI].readHexChar shl 4 or hex[2*i + 1 + offI].readHexChar + + # write single nibble from odd length hex + if (offO > 0) and (offI > 0): + res[offO-1] = hex[offI-1].readHexChar + +proc getValue*(key: string, outVal: var string) {.raises: [ValueError].} = + let hex = os.getEnv(key) + let size = (hex.len div 2) + (hex.len and 0x01) + outVal.setLen(size) + decodePaddedHex(hex, cast[ptr UncheckedArray[byte]](outVal[0].addr), size) + +proc getValue*[T: SomePrimitives](key: string, outVal: var seq[T]) = + let hex = os.getEnv(key) + let byteSize = (hex.len div 2) + (hex.len and 0x01) + let size = (byteSize + sizeof(T) - 1) div sizeof(T) + outVal.setLen(size) + decodePaddedHex(hex, cast[ptr UncheckedArray[byte]](outVal[0].addr), size * sizeof(T)) + +proc getValue*[N, T: SomePrimitives](key: string, outVal: var array[N, T]) = + let hex = os.getEnv(key) + decodePaddedHex(hex, cast[ptr UncheckedArray[byte]](outVal[0].addr), sizeof(outVal)) + +proc getValue*(key: string, outVal: var SomePrimitives) {.raises: [ValueError].} = + let hex = os.getEnv(key) + decodePaddedHex(hex, cast[ptr UncheckedArray[byte]](outVal.addr), sizeof(outVal)) + +template uTypeIsPrimitives*[T](_: type seq[T]): bool = + when T is SomePrimitives: + true + else: + false + +template uTypeIsPrimitives*[N, T](_: type array[N, T]): bool = + when T is SomePrimitives: + true + else: + false + +template uTypeIsPrimitives*[T](_: type openArray[T]): bool = + when T is SomePrimitives: + true + else: + false + +template uTypeIsRecord*(_: typed): bool = + false + +template uTypeIsRecord*[T](_: type seq[T]): bool = + when T is (object or tuple): + true + else: + false + +template uTypeIsRecord*[N, T](_: type array[N, T]): bool = + when T is (object or tuple): + true + else: + false + + +func constructKey*(prefix: string, keys: openArray[string]): string = + var newKey: string + + let envvarPrefix = prefix.strip().toUpper().multiReplace(("-", "_"), (" ", "_")) + newKey.add(envvarPrefix) + + + for k in keys: + newKey.add("_") + + let envvarKey = k.toUpper().multiReplace(("-", "_"), (" ", "_")) + newKey.add(envvarKey) + + newKey diff --git a/waku/common/envvar_serialization/writer.nim b/waku/common/envvar_serialization/writer.nim new file mode 100644 index 0000000000..2b160e1fcf --- /dev/null +++ b/waku/common/envvar_serialization/writer.nim @@ -0,0 +1,54 @@ +import + typetraits, options, tables, os, + serialization, ./utils + +type + EnvvarWriter* = object + prefix: string + key: seq[string] + +proc init*(T: type EnvvarWriter, prefix: string): T = + result.prefix = prefix + +proc writeValue*(w: var EnvvarWriter, value: auto) = + mixin enumInstanceSerializedFields, writeValue, writeFieldIMPL + # TODO: reduce allocation + + when value is string: + let key = constructKey(w.prefix, w.key) + os.putEnv(key, value) + + elif value is (SomePrimitives or range): + let key = constructKey(w.prefix, w.key) + setValue(key, value) + + elif value is Option: + if value.isSome: + w.writeValue value.get + + elif value is (seq or array or openArray): + when uTypeIsPrimitives(type value): + let key = constructKey(w.prefix, w.key) + setValue(key, value) + + elif uTypeIsRecord(type value): + let key = w.key[^1] + for i in 0..