From 46a197e48b511b5d072524c10e57a85b23499f87 Mon Sep 17 00:00:00 2001 From: Nikita Melkozerov Date: Mon, 24 Jul 2023 12:35:30 +0200 Subject: [PATCH] [resotocore][fix] Update config override docs and fix the validation bug (#1746) --- .../config/config_override_service.py | 8 +++++-- resotocore/resotocore/system_start.py | 18 +++++++-------- .../config/config_override_service_test.py | 22 +++++++++++++++++-- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/resotocore/resotocore/config/config_override_service.py b/resotocore/resotocore/config/config_override_service.py index 8a528527f7..f3b7124ab1 100644 --- a/resotocore/resotocore/config/config_override_service.py +++ b/resotocore/resotocore/config/config_override_service.py @@ -53,7 +53,7 @@ def coerce_and_validate(self, config: Json, model: Model) -> Json: if key in model: try: value_kind = model[key] - coerced = value_kind.check_valid(value, normalize=False, config_context=True) + coerced = value_kind.check_valid(value, normalize=False, config_context=True, ignore_missing=True) final_config[key] = value_kind.sort_json(coerced or value) except Exception as ex: raise AttributeError(f"Error validating section {key}: {ex}") from ex @@ -75,10 +75,13 @@ async def _get_overrides(self, silent: bool = False) -> Dict[ConfigId, Json]: [file for file in path.iterdir() if file.is_file() and file.suffix in (".yml", ".yaml", ".json")] ) elif path.suffix in (".yml", ".yaml", ".json"): - config_files.append(path) + config_files.append(path.expanduser()) else: log.warning(f"Config override path {path} is neither a directory nor a yaml/json file, skipping.") + if config_files: + log.info("Loading config overrides from: %s", ", ".join(str(file) for file in config_files)) + # json with all merged overrides for all components such as resotocore, resotoworker, etc. overrides_json: Json = {} @@ -118,6 +121,7 @@ def is_dict(config_id: str, obj: Any) -> bool: log.warning("No config overrides found.") return {} + log.info("Loaded config overrides for: %s", ", ".join(str(config_id) for config_id in all_config_overrides)) return all_config_overrides def get_override(self, config_id: ConfigId) -> Optional[Json]: diff --git a/resotocore/resotocore/system_start.py b/resotocore/resotocore/system_start.py index b47c974970..91f7412279 100644 --- a/resotocore/resotocore/system_start.py +++ b/resotocore/resotocore/system_start.py @@ -186,16 +186,16 @@ def key_value(kv: str) -> Tuple[str, JsonElement]: help="Override configuration parameters via a YAML file or directory with YAML files. " "The existing configuration will be patched with the provided values. " "Note: this argument allows multiple overrides separated by space, in this case the " - "resulting configuration will be the merge of all the provided files, in the order they are provided." - "The same section can be overridden multiple times, in this case the last override will be used." - "Example: --override-path /path/to/config/dir/ /path/to/your/config.yaml" - "Be sure to specify the correct config id in the yaml, e.g. override for resotoworker would looke like:\n" + "resulting configuration will be the merge of all the provided files, in the order they are provided. " + "The same section can be overridden multiple times, in this case the last override will be used. " + "Example: --override-path /path/to/config/dir/ /path/to/your/config.yaml " + "Be sure to specify the correct config id in the name of the yaml file, e.g. override for resotoworker would " + "be called resoto.worker.yaml and look like:\n" """ -resoto.worker: - resotoworker: - ... - aws: - ...""", +resotoworker: + ... +aws: + ...""", ) parser.add_argument( "--verbose", "-v", dest="verbose", default=False, action="store_true", help="Enable verbose logging." diff --git a/resotocore/tests/resotocore/config/config_override_service_test.py b/resotocore/tests/resotocore/config/config_override_service_test.py index 0c4d67fd2e..0cf12f3a8e 100644 --- a/resotocore/tests/resotocore/config/config_override_service_test.py +++ b/resotocore/tests/resotocore/config/config_override_service_test.py @@ -125,10 +125,12 @@ async def test_validation() -> None: encoding="utf-8", ) - foo = ComplexKind("foo", [], [Property("bar", "int32"), Property("kind", "string")]) + foo_kinds = ComplexKind( + "foo", [], [Property("bar", "int32", required=True), Property("kind", "string", required=True)] + ) async def get_configs_model() -> Model: - return Model.from_kinds([foo]) + return Model.from_kinds([foo_kinds]) override_service = ConfigOverrideService([Path(tmp)], get_configs_model) await override_service.load() @@ -154,6 +156,22 @@ async def get_configs_model() -> Model: # invalid config should be ignored assert override_service.get_override(ConfigId("resoto.core")) is None + resotocore_1_conf.write_text( + """ +foo: + bar: 42""", + encoding="utf-8", + ) + override_service = ConfigOverrideService([Path(tmp)], get_configs_model) + await override_service.load() + + # missing keys are allowed for overrides + assert override_service.get_override(ConfigId("resoto.core")) == { + "foo": { + "bar": 42, + } + } + @pytest.mark.asyncio async def test_load_json() -> None: