From ea6f107bf0493ae7b12d00702d66fadc7fa85cb0 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Sat, 2 Dec 2023 15:28:01 +0000 Subject: [PATCH 01/28] don't warn about missing findpackage --- config_utilities/cmake/OptionalPackage.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config_utilities/cmake/OptionalPackage.cmake b/config_utilities/cmake/OptionalPackage.cmake index 83fd50b..33eb365 100644 --- a/config_utilities/cmake/OptionalPackage.cmake +++ b/config_utilities/cmake/OptionalPackage.cmake @@ -8,7 +8,7 @@ # not the package should be used macro(FIND_OPTIONAL package_name package_enabled) if(${package_enabled}) - find_package(${package_name}) + find_package(${package_name} QUIET) endif() if(${${package_name}_FOUND}) From f3f63ee908b104aede1615b608dc194b25c7eefc Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Sat, 2 Dec 2023 15:52:48 +0000 Subject: [PATCH 02/28] (wip) start on ros2 interface --- config_utilities/CMakeLists.txt | 7 + .../cmake/config_utilitiesConfig.cmake.in | 10 +- .../include/config_utilities/parsing/ros2.h | 141 ++++++++++++++++++ 3 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 config_utilities/include/config_utilities/parsing/ros2.h diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index 8b5faa8..4d8e789 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -11,6 +11,7 @@ include(OptionalPackage) option(BUILD_SHARED_LIBS "Build shared libs" ON) option(CONFIG_UTILS_ENABLE_ROS "Export roscpp and build related code" ON) +option(CONFIG_UTILS_ENABLE_ROS2 "Export rclcpp and build related code" ON) option(CONFIG_UTILS_ENABLE_EIGEN "Export Eigen and build related code" ON) option(CONFIG_UTILS_ENABLE_GLOG "Export glog and build related code" ON) option(CONFIG_UTILS_BUILD_TESTS "Build unit tests" ON) @@ -20,6 +21,7 @@ option(CONFIG_UTILS_BUILD_DEMOS "Build demo executables" ON) find_package(yaml-cpp REQUIRED) find_package(Boost REQUIRED COMPONENTS filesystem system) find_optional(roscpp CONFIG_UTILS_ENABLE_ROS) +find_optional(rclcpp CONFIG_UTILS_ENABLE_ROS2) find_optional(Eigen3 CONFIG_UTILS_ENABLE_EIGEN) find_optional_pkgcfg(libglog CONFIG_UTILS_ENABLE_GLOG) @@ -57,6 +59,10 @@ if(ENABLE_roscpp) target_include_directories(${PROJECT_NAME} PUBLIC ${roscpp_INCLUDE_DIRS}) endif() +if(ENABLE_rclcpp) + target_link_libraries(${PROJECT_NAME} PUBLIC rclcpp::rclcpp) +endif() + if(ENABLE_Eigen3) target_link_libraries(${PROJECT_NAME} PUBLIC Eigen3::Eigen) endif() @@ -67,6 +73,7 @@ endif() message(STATUS "Eigen features enabled: ${ENABLE_Eigen3}") message(STATUS "ROS features enabled: ${ENABLE_roscpp}") +message(STATUS "ROS2 features enabled: ${ENABLE_rclcpp}") message(STATUS "glog features enabled: ${ENABLE_libglog}") if(CONFIG_UTILS_BUILD_DEMOS) diff --git a/config_utilities/cmake/config_utilitiesConfig.cmake.in b/config_utilities/cmake/config_utilitiesConfig.cmake.in index 756a3cf..0d0e3f6 100644 --- a/config_utilities/cmake/config_utilitiesConfig.cmake.in +++ b/config_utilities/cmake/config_utilitiesConfig.cmake.in @@ -1,6 +1,7 @@ @PACKAGE_INIT@ -get_filename_component(config_utilities_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +get_filename_component(config_utilities_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" + PATH) include(CMakeFindDependencyMacro) find_dependency(yaml-cpp REQUIRED) @@ -11,6 +12,11 @@ endif() if(@ENABLE_roscpp@) find_dependency(roscpp REQUIRED) + set(config_utilities_FOUND_CATKIN_PROJECT TRUE) +endif() + +if(@ENABLE_rclcpp@) + find_dependency(rclcpp REQUIRED) endif() if(@ENABLE_libglog@) @@ -24,6 +30,4 @@ endif() set(config_utilities_LIBRARIES config_utilities::config_utilities) -set(config_utilities_FOUND_CATKIN_PROJECT TRUE) - check_required_components(config_utilities) diff --git a/config_utilities/include/config_utilities/parsing/ros2.h b/config_utilities/include/config_utilities/parsing/ros2.h new file mode 100644 index 0000000..3f820f6 --- /dev/null +++ b/config_utilities/include/config_utilities/parsing/ros2.h @@ -0,0 +1,141 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#pragma once + +#include +#include +#include +#include + +#include + +#include "config_utilities/factory.h" +#include "config_utilities/internal/string_utils.h" +#include "config_utilities/internal/visitor.h" +#include "config_utilities/internal/yaml_utils.h" +#include "config_utilities/parsing/yaml.h" + +namespace config { + +namespace internal { + +// Translate the parameter server to yaml code. This is inlined to shift the dependency on ROS to the downstream cpp +// file. +inline YAML::Node xmlRpcToYaml(const XmlRpc::XmlRpcValue& xml) { + switch (xml.getType()) { + case XmlRpc::XmlRpcValue::Type::TypeBoolean: + return YAML::Node(static_cast(xml)); + case XmlRpc::XmlRpcValue::Type::TypeInt: + return YAML::Node(static_cast(xml)); + case XmlRpc::XmlRpcValue::Type::TypeDouble: + return YAML::Node(static_cast(xml)); + case XmlRpc::XmlRpcValue::Type::TypeString: + return YAML::Node(static_cast(xml)); + case XmlRpc::XmlRpcValue::Type::TypeArray: { + YAML::Node node(YAML::NodeType::Sequence); + for (int i = 0; i < xml.size(); ++i) { + node.push_back(xmlRpcToYaml(xml[i])); + } + return node; + } + case XmlRpc::XmlRpcValue::Type::TypeStruct: { + YAML::Node node(YAML::NodeType::Map); + for (auto it = xml.begin(); it != xml.end(); ++it) { + node[it->first] = xmlRpcToYaml(it->second); + } + return node; + } + default: + return YAML::Node(); + } +} + +inline YAML::Node rosToYaml(const rclcpp::Node& node) { + YAML::Node root; + + auto all_params = node.list_parameters({}, rcl_interfaces::srv::ListParameters::Request::DEPTH_RECURSIVE); + + for (std::string& name : all_params.names) { + // TODO(nathan) optional namespace filter + + name = name.erase(0, nh.getNamespace().length()); // Remove the nodehandle's namespace. + std::vector name_parts = splitNamespace(name); + std::string local_name = ""; + if (!name_parts.empty()) { + name = name.substr(1); // Remove the leading slash. + local_name = name_parts.back(); + name_parts.pop_back(); + } + + const auto sub_namespace = joinNamespace(name_parts); + + // Get the Xml Value + XmlRpc::XmlRpcValue value; + nh.getParam(name, value); + + // Convert data to yaml. + YAML::Node local_node; + if (local_name.empty()) { + local_node = xmlRpcToYaml(value); + } else { + local_node[local_name] = xmlRpcToYaml(value); + } + moveDownNamespace(local_node, sub_namespace); + mergeYamlNodes(root, local_node); + } + + return root; +} + +} // namespace internal + +/** + * @brief Loads a config from a yaml node. + * + * @tparam ConfigT The config type. This can also be a VirtualConfig or a std::vector. + * @param nh The ROS nodehandle to create the config from. + * @param name_space Optionally specify a name space to create the config from. Separate names with slashes '/'. + * Example: "my_config/my_sub_config". + * @returns The config. + */ +template +ConfigT fromRos(const ros::NodeHandle& nh, const std::string& name_space = "") { + const ros::NodeHandle ns_nh = ros::NodeHandle(nh, name_space); + const YAML::Node node = internal::rosToYaml(ns_nh); + return internal::fromYamlImpl(node, "", static_cast(nullptr)); +} + +} // namespace config From 19fe9744c9a768bc144bee750da7fd8e4b957a23 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Wed, 10 Jul 2024 10:05:26 -0400 Subject: [PATCH 03/28] add new launch extension package --- config_utilities_launch/.flake8 | 5 +++ config_utilities_launch/LICENSE | 25 +++++++++++ .../config_utilities_launch/__init__.py | 45 +++++++++++++++++++ config_utilities_launch/package.xml | 18 ++++++++ .../resource/config_utilities_launch | 0 config_utilities_launch/setup.cfg | 4 ++ config_utilities_launch/setup.py | 25 +++++++++++ .../test/test_copyright.py | 25 +++++++++++ config_utilities_launch/test/test_flake8.py | 25 +++++++++++ config_utilities_launch/test/test_pep257.py | 23 ++++++++++ config_utilities_launch/test_launch.yaml | 13 ++++++ 11 files changed, 208 insertions(+) create mode 100644 config_utilities_launch/.flake8 create mode 100644 config_utilities_launch/LICENSE create mode 100644 config_utilities_launch/config_utilities_launch/__init__.py create mode 100644 config_utilities_launch/package.xml create mode 100644 config_utilities_launch/resource/config_utilities_launch create mode 100644 config_utilities_launch/setup.cfg create mode 100644 config_utilities_launch/setup.py create mode 100644 config_utilities_launch/test/test_copyright.py create mode 100644 config_utilities_launch/test/test_flake8.py create mode 100644 config_utilities_launch/test/test_pep257.py create mode 100644 config_utilities_launch/test_launch.yaml diff --git a/config_utilities_launch/.flake8 b/config_utilities_launch/.flake8 new file mode 100644 index 0000000..5768200 --- /dev/null +++ b/config_utilities_launch/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203 +per-file-ignores = + *__init__.py:F401,F403 diff --git a/config_utilities_launch/LICENSE b/config_utilities_launch/LICENSE new file mode 100644 index 0000000..574ef07 --- /dev/null +++ b/config_utilities_launch/LICENSE @@ -0,0 +1,25 @@ +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/config_utilities_launch/config_utilities_launch/__init__.py b/config_utilities_launch/config_utilities_launch/__init__.py new file mode 100644 index 0000000..19160c2 --- /dev/null +++ b/config_utilities_launch/config_utilities_launch/__init__.py @@ -0,0 +1,45 @@ +"""Main entry point for the config_utilities_launch package.""" + +from launch.action import Action +from launch.frontend import expose_action, Entity, Parser +from launch.launch_context import LaunchContext +from launch.utilities import perform_substitutions, normalize_to_list_of_substitutions +from launch_ros.actions import Node + +from typing import Optional, List + + +def _substitute(context, value): + return perform_substitutions(context, normalize_to_list_of_substitutions(value)) + + +@expose_action("configured_node") +class ConfiguredNode(Node): + """Configurable node via ytt.""" + + def __init__(self, *args, config_files=None, config_options=None, **kwargs): + """Construct a node.""" + super().__init__(*args, **kwargs) + self._config_files = config_files if config_files is not None else [] + self._config_options = config_options if config_options is not None else [] + + @classmethod + def parse(cls, entity: Entity, parser: Parser): + """Parse a node from a launch file.""" + _, kwargs = super().parse(entity, parser) + files = entity.get_attr("config_files", optional=True, data_type=List[str]) + if files is not None: + kwargs["config_files"] = [parser.parse_substitution(x) for x in files] + + opts = entity.get_attr("config_options", optional=True, data_type=List[str]) + if opts is not None: + kwargs["config_options"] = [parser.parse_substitution(x) for x in opts] + + return cls, kwargs + + def execute(self, context: LaunchContext) -> Optional[List[Action]]: + """Call ytt and then call the underlying execute.""" + self._config_files = [_substitute(context, x) for x in self._config_files] + self._config_options = [_substitute(context, x) for x in self._config_options] + args = ["ytt"] + [] + return super().execute(context) diff --git a/config_utilities_launch/package.xml b/config_utilities_launch/package.xml new file mode 100644 index 0000000..74b4c5d --- /dev/null +++ b/config_utilities_launch/package.xml @@ -0,0 +1,18 @@ + + + + config_utilities_launch + 0.0.0 + TODO: Package description + nathan + BSD-3-Clause + + ament_copyright + ament_flake8 + ament_pep257 + python3-pytest + + + ament_python + + diff --git a/config_utilities_launch/resource/config_utilities_launch b/config_utilities_launch/resource/config_utilities_launch new file mode 100644 index 0000000..e69de29 diff --git a/config_utilities_launch/setup.cfg b/config_utilities_launch/setup.cfg new file mode 100644 index 0000000..9b4b8d3 --- /dev/null +++ b/config_utilities_launch/setup.cfg @@ -0,0 +1,4 @@ +[develop] +script_dir=$base/lib/config_utilities_launch +[install] +install_scripts=$base/lib/config_utilities_launch diff --git a/config_utilities_launch/setup.py b/config_utilities_launch/setup.py new file mode 100644 index 0000000..cd48204 --- /dev/null +++ b/config_utilities_launch/setup.py @@ -0,0 +1,25 @@ +from setuptools import find_packages, setup + +package_name = "config_utilities_launch" + +setup( + name=package_name, + version="0.0.0", + packages=find_packages(exclude=["test"]), + data_files=[ + ("share/ament_index/resource_index/packages", ["resource/" + package_name]), + ("share/" + package_name, ["package.xml"]), + ], + install_requires=["setuptools"], + zip_safe=True, + maintainer="nathan", + maintainer_email="nathan.h.hughes@gmail.com", + description="TODO: Package description", + license="BSD-3-Clause", + tests_require=["pytest"], + entry_points={ + "launch.frontend.launch_extension": [ + "config_utilities_launch = config_utilities_launch" + ], + }, +) diff --git a/config_utilities_launch/test/test_copyright.py b/config_utilities_launch/test/test_copyright.py new file mode 100644 index 0000000..97a3919 --- /dev/null +++ b/config_utilities_launch/test/test_copyright.py @@ -0,0 +1,25 @@ +# Copyright 2015 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_copyright.main import main +import pytest + + +# Remove the `skip` decorator once the source file(s) have a copyright header +@pytest.mark.skip(reason='No copyright header has been placed in the generated source file.') +@pytest.mark.copyright +@pytest.mark.linter +def test_copyright(): + rc = main(argv=['.', 'test']) + assert rc == 0, 'Found errors' diff --git a/config_utilities_launch/test/test_flake8.py b/config_utilities_launch/test/test_flake8.py new file mode 100644 index 0000000..27ee107 --- /dev/null +++ b/config_utilities_launch/test/test_flake8.py @@ -0,0 +1,25 @@ +# Copyright 2017 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_flake8.main import main_with_errors +import pytest + + +@pytest.mark.flake8 +@pytest.mark.linter +def test_flake8(): + rc, errors = main_with_errors(argv=[]) + assert rc == 0, \ + 'Found %d code style errors / warnings:\n' % len(errors) + \ + '\n'.join(errors) diff --git a/config_utilities_launch/test/test_pep257.py b/config_utilities_launch/test/test_pep257.py new file mode 100644 index 0000000..b234a38 --- /dev/null +++ b/config_utilities_launch/test/test_pep257.py @@ -0,0 +1,23 @@ +# Copyright 2015 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_pep257.main import main +import pytest + + +@pytest.mark.linter +@pytest.mark.pep257 +def test_pep257(): + rc = main(argv=['.', 'test']) + assert rc == 0, 'Found code style errors / warnings' diff --git a/config_utilities_launch/test_launch.yaml b/config_utilities_launch/test_launch.yaml new file mode 100644 index 0000000..04286bc --- /dev/null +++ b/config_utilities_launch/test_launch.yaml @@ -0,0 +1,13 @@ +--- +launch: + - configured_node: + pkg: turtlesim + exec: turtlesim_node + name: sim + namespace: turtlesim1 + config_files: + - $(find-pkg-share config_utilities_launch)/some_param + - $(find-pkg-prefix config_utilities_launch)/some_other_param + config_options: + - --foo + - -bar From 3a3c15648f41d3ece99da46a4aef565bcfb6eb2f Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Wed, 10 Jul 2024 10:05:50 -0400 Subject: [PATCH 04/28] drop broken unit test install --- config_utilities/CMakeLists.txt | 1 - config_utilities/test/CMakeLists.txt | 7 ------- 2 files changed, 8 deletions(-) diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index 4d8e789..cadef7f 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -15,7 +15,6 @@ option(CONFIG_UTILS_ENABLE_ROS2 "Export rclcpp and build related code" ON) option(CONFIG_UTILS_ENABLE_EIGEN "Export Eigen and build related code" ON) option(CONFIG_UTILS_ENABLE_GLOG "Export glog and build related code" ON) option(CONFIG_UTILS_BUILD_TESTS "Build unit tests" ON) -option(CONFIG_UTILS_INSTALL_TESTS "Install test executable" ON) option(CONFIG_UTILS_BUILD_DEMOS "Build demo executables" ON) find_package(yaml-cpp REQUIRED) diff --git a/config_utilities/test/CMakeLists.txt b/config_utilities/test/CMakeLists.txt index 2f0061e..fc09fbd 100644 --- a/config_utilities/test/CMakeLists.txt +++ b/config_utilities/test/CMakeLists.txt @@ -34,10 +34,3 @@ target_include_directories(test_${PROJECT_NAME} PRIVATE include) target_link_libraries(test_${PROJECT_NAME} PRIVATE ${PROJECT_NAME} GTest::gtest_main) gtest_add_tests(TARGET test_${PROJECT_NAME}) - -if(${CONFIG_UTILS_INSTALL_TESTS}) - # note that ros uses libdir to handle finding executables by package, but this - # isn't an ideal install location normally - install(TARGETS test_${PROJECT_NAME} - RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}) -endif() From 2a3ac3dfbc1bb65943bb316b0d54bbe8b904a236 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Sat, 30 Nov 2024 11:49:21 -0500 Subject: [PATCH 05/28] switch config tag format and work on parsing correctly --- .../config_utilities_launch/__init__.py | 80 +++++++++++++++---- config_utilities_launch/setup.py | 1 - 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/config_utilities_launch/config_utilities_launch/__init__.py b/config_utilities_launch/config_utilities_launch/__init__.py index 19160c2..eaf7e88 100644 --- a/config_utilities_launch/config_utilities_launch/__init__.py +++ b/config_utilities_launch/config_utilities_launch/__init__.py @@ -7,39 +7,89 @@ from launch_ros.actions import Node from typing import Optional, List +from dataclasses import dataclass + + +@dataclass +class ConfigFile: + """Quick class representing a file.""" + + filepath: str # TODO(nathan) this isn't just a str + ns: Optional[str] = None # TODO(nathan) this isn't just a str + allow_substs: bool = False + + +@dataclass +class ConfigField: + """Quick class representing a single config value at a name.""" + + name: str + value: str # TODO(nathan) this isn't just a str + allow_substs: bool = False def _substitute(context, value): return perform_substitutions(context, normalize_to_list_of_substitutions(value)) +def _parse_config_section(section, parser): + name_attr = section.get_attr("name", optional=True) + from_attr = section.get_attr("from", optional=True) + allow_attr = section.get_attr("allow_substs", optional=True) + ns_attr = section.get_attr("ns", optional=True) if from_attr is not None else None + value_attr = ( + section.get_attr("value", optional=True) if name_attr is not None else None + ) + + if from_attr is not None and name_attr is not None: + raise RuntimeError( + "Invalid config field: must either be a named param or a file" + ) + + if from_attr is None and name_attr is None: + raise RuntimeError("Invalid config field: must have a 'name' or 'from' tag") + + section.assert_entity_completely_parsed() + # TODO(nathan) validate this parses bools correctly + allow_substs = False if allow_attr is not None else bool(allow_attr) + + if from_attr is not None: + from_attr = parser.parse_substitution(from_attr) + if ns_attr is not None: + ns_attr = parser.parse_substitution(ns_attr) + + return ConfigFile(filepath=from_attr, ns=ns_attr, allow_substs=allow_substs) + else: + value_attr = parser.parse_substitution(value_attr) + return ConfigField(name=name_attr, value=value_attr, allow_substs=allow_substs) + + +def _parse_config_list(config, parser): + return [_parse_config_section(x, parser) for x in config] + + @expose_action("configured_node") class ConfiguredNode(Node): - """Configurable node via ytt.""" + """Extension of Node that allows for collating arbitrary YAML files.""" - def __init__(self, *args, config_files=None, config_options=None, **kwargs): + def __init__(self, *args, config=None, **kwargs): """Construct a node.""" super().__init__(*args, **kwargs) - self._config_files = config_files if config_files is not None else [] - self._config_options = config_options if config_options is not None else [] + self._config = config @classmethod def parse(cls, entity: Entity, parser: Parser): """Parse a node from a launch file.""" _, kwargs = super().parse(entity, parser) - files = entity.get_attr("config_files", optional=True, data_type=List[str]) - if files is not None: - kwargs["config_files"] = [parser.parse_substitution(x) for x in files] - - opts = entity.get_attr("config_options", optional=True, data_type=List[str]) - if opts is not None: - kwargs["config_options"] = [parser.parse_substitution(x) for x in opts] + config = entity.get_attr("config", optional=True, data_type=List[Entity]) + if config is not None: + kwargs["config"] = _parse_config_list(config, parser) return cls, kwargs def execute(self, context: LaunchContext) -> Optional[List[Action]]: - """Call ytt and then call the underlying execute.""" - self._config_files = [_substitute(context, x) for x in self._config_files] - self._config_options = [_substitute(context, x) for x in self._config_options] - args = ["ytt"] + [] + """Collate parsed config entries and pass to node.""" + print(self._config) + self._config = [_substitute(context, x) for x in self._config] + print(self._config) return super().execute(context) diff --git a/config_utilities_launch/setup.py b/config_utilities_launch/setup.py index cd48204..8df0b49 100644 --- a/config_utilities_launch/setup.py +++ b/config_utilities_launch/setup.py @@ -11,7 +11,6 @@ ("share/" + package_name, ["package.xml"]), ], install_requires=["setuptools"], - zip_safe=True, maintainer="nathan", maintainer_email="nathan.h.hughes@gmail.com", description="TODO: Package description", From a6e389d353f8a5db3fc4779beb567f43f2a27828 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Mon, 2 Dec 2024 15:02:43 -0500 Subject: [PATCH 06/28] work on performing substitutions on parsed param fields --- .../config_utilities_launch/__init__.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/config_utilities_launch/config_utilities_launch/__init__.py b/config_utilities_launch/config_utilities_launch/__init__.py index eaf7e88..89351a1 100644 --- a/config_utilities_launch/config_utilities_launch/__init__.py +++ b/config_utilities_launch/config_utilities_launch/__init__.py @@ -29,7 +29,19 @@ class ConfigField: def _substitute(context, value): - return perform_substitutions(context, normalize_to_list_of_substitutions(value)) + match value: + case ConfigFile(filepath, ns, allow_substs): + if allow_substs: + filepath = perform_substitutions( + context, normalize_to_list_of_substitutions(filepath) + ) + return {ns: filepath} + case ConfigField(name, value, allow_substs): + if allow_substs: + value = perform_substitutions( + context, normalize_to_list_of_substitutions(value) + ) + return {name: value} def _parse_config_section(section, parser): From a74c0cb71fc7b535b7ab9994efe4889aae5516aa Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Mon, 23 Dec 2024 09:36:38 -0500 Subject: [PATCH 07/28] allow optional new YAML merging behavior --- .../config_utilities/internal/yaml_utils.h | 5 +- config_utilities/src/yaml_utils.cpp | 57 ++++++++++++------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/config_utilities/include/config_utilities/internal/yaml_utils.h b/config_utilities/include/config_utilities/internal/yaml_utils.h index 2d0fe70..2e35db9 100644 --- a/config_utilities/include/config_utilities/internal/yaml_utils.h +++ b/config_utilities/include/config_utilities/internal/yaml_utils.h @@ -44,9 +44,10 @@ namespace config::internal { /** * @brief Merges node b into a, overwriting values previously defined in a if they can not be - * merged. Modifies node a, whereas b is const. + * merged. Modifies node a, whereas b is const. Sequences can optionally be appended together at the same level of the + * YAML tree. */ -void mergeYamlNodes(YAML::Node& a, const YAML::Node& b); +void mergeYamlNodes(YAML::Node& a, const YAML::Node& b, bool extend_sequences = false); /** * @brief Get a pointer to the final node of the specified namespace if it exists, where each map in the yaml is diff --git a/config_utilities/src/yaml_utils.cpp b/config_utilities/src/yaml_utils.cpp index 6eb5ae9..1c5f04c 100644 --- a/config_utilities/src/yaml_utils.cpp +++ b/config_utilities/src/yaml_utils.cpp @@ -38,37 +38,50 @@ #include "config_utilities/internal/string_utils.h" namespace config::internal { +namespace { -void mergeYamlNodes(YAML::Node& a, const YAML::Node& b) { - if (!b.IsMap()) { - // If b is not a map, merge result is b, unless b is null. - if (b.IsNull() || !b.IsDefined()) { - return; - } - a = YAML::Clone(b); +inline void mergeLeaves(YAML::Node& a, const YAML::Node& b, bool extend_sequences) { + // If b is invalid, we can't do anything. + if (b.IsNull() || !b.IsDefined()) { return; } - if (!a.IsMap()) { - // If a is not a map, merge result is b + + // If either a or b is not a sequence, assign b to a (or if extending is disabled) + if (!extend_sequences || !a.IsSequence() || !b.IsSequence()) { a = YAML::Clone(b); return; } - if (!b.size()) { - // If a is a map, and b is an empty map, return a + + // both a and b are sequences: append b to a. + for (const auto& child : b) { + a.push_back(YAML::Clone(child)); + } +} + +} // namespace + +void mergeYamlNodes(YAML::Node& a, const YAML::Node& b, bool extend_sequences) { + // If either node is a leaf in the config tree, pass merging behavior to helper function + if (!b.IsMap() || !a.IsMap()) { + mergeLeaves(a, b, extend_sequences); return; } - // Merge all entries of b into a. - for (const auto kv_pair : b) { - if (kv_pair.first.IsScalar()) { - const std::string& key = kv_pair.first.Scalar(); - if (a[key]) { - // Node exists. Merge recursively. - YAML::Node a_sub = a[key]; // This node is a ref. - mergeYamlNodes(a_sub, kv_pair.second); - } else { - a[key] = YAML::Clone(kv_pair.second); - } + // Both a and b are maps: merge all entries of b into a. + for (const auto& node : b) { + if (!node.first.IsScalar()) { + // TODO(nathan) warn about not handling complex keys + continue; + } + + const auto& key = node.first.Scalar(); + if (a[key]) { + // Node exists. Merge recursively. + YAML::Node a_sub = a[key]; // This node is a ref. + mergeYamlNodes(a_sub, node.second); + } else { + // Leaf of a, but b continues: insert b + a[key] = YAML::Clone(node.second); } } } From 832edfa37149842d374f13de0090c9d2dd2d5fad Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Thu, 2 Jan 2025 11:03:56 -0500 Subject: [PATCH 08/28] (wip) start on context --- config_utilities/CMakeLists.txt | 1 + .../config_utilities/internal/context.h | 60 +++++++++++++++++++ config_utilities/src/context.cpp | 55 +++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 config_utilities/include/config_utilities/internal/context.h create mode 100644 config_utilities/src/context.cpp diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index cadef7f..e17f55a 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -27,6 +27,7 @@ find_optional_pkgcfg(libglog CONFIG_UTILS_ENABLE_GLOG) add_library( ${PROJECT_NAME} src/asl_formatter.cpp + src/context.cpp src/conversions.cpp src/external_registry.cpp src/factory.cpp diff --git a/config_utilities/include/config_utilities/internal/context.h b/config_utilities/include/config_utilities/internal/context.h new file mode 100644 index 0000000..e435eaf --- /dev/null +++ b/config_utilities/include/config_utilities/internal/context.h @@ -0,0 +1,60 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#pragma once + +#include + +#include + +namespace config::internal { + +/** + * @brief Context is a singleton that holds the raw parsed information used to generate configs + */ +class Context { + public: + ~Context() = default; + + static void update(const YAML::Node& other, const std::string& ns); + + private: + Context() = default; + static Context& instance(); + + YAML::Node contents_; +}; + +} // namespace config::internal diff --git a/config_utilities/src/context.cpp b/config_utilities/src/context.cpp new file mode 100644 index 0000000..f7ea373 --- /dev/null +++ b/config_utilities/src/context.cpp @@ -0,0 +1,55 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/internal/context.h" + +#include "config_utilities/internal/yaml_utils.h" + +namespace config::internal { + +Context& Context::instance() { + static Context s_instance; + return s_instance; +} + +void Context::update(const YAML::Node& other, const std::string& ns) { + auto& context = instance(); + auto node = YAML::Clone(other); + moveDownNamespace(node, ns); + // default behavior of context is to act like the ROS1 param server and extend sequences + mergeYamlNodes(context.contents_, node, true); +} + +} // namespace config::internal From 803ea39007f515381b29d5adc1e3704f5dca9ead Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Thu, 2 Jan 2025 16:44:31 +0000 Subject: [PATCH 09/28] outline basic commandline infrastructure --- config_utilities/CMakeLists.txt | 1 + .../internal/commandline_utils.h | 49 ++++++ .../config_utilities/parsing/commandline.h | 114 ++++++++++++++ .../include/config_utilities/parsing/ros2.h | 141 ------------------ config_utilities/src/commandline_utils.cpp | 45 ++++++ 5 files changed, 209 insertions(+), 141 deletions(-) create mode 100644 config_utilities/include/config_utilities/internal/commandline_utils.h create mode 100644 config_utilities/include/config_utilities/parsing/commandline.h delete mode 100644 config_utilities/include/config_utilities/parsing/ros2.h create mode 100644 config_utilities/src/commandline_utils.cpp diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index e17f55a..3d5c5c4 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -27,6 +27,7 @@ find_optional_pkgcfg(libglog CONFIG_UTILS_ENABLE_GLOG) add_library( ${PROJECT_NAME} src/asl_formatter.cpp + src/commandline_utils.cpp src/context.cpp src/conversions.cpp src/external_registry.cpp diff --git a/config_utilities/include/config_utilities/internal/commandline_utils.h b/config_utilities/include/config_utilities/internal/commandline_utils.h new file mode 100644 index 0000000..216c37f --- /dev/null +++ b/config_utilities/include/config_utilities/internal/commandline_utils.h @@ -0,0 +1,49 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#pragma once + +#include + +namespace config::internal { + +/** + * @brief Parse and collate YAML node from arguments, optionally removing arguments + * @param argc Number of command line arguments + * @param argv Command line argument strings + */ +YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args); + +} // namespace config::internal diff --git a/config_utilities/include/config_utilities/parsing/commandline.h b/config_utilities/include/config_utilities/parsing/commandline.h new file mode 100644 index 0000000..a2c7235 --- /dev/null +++ b/config_utilities/include/config_utilities/parsing/commandline.h @@ -0,0 +1,114 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#pragma once + +#include +#include +#include +#include + +#include "config_utilities/factory.h" +#include "config_utilities/internal/commandline_utils.h" +#include "config_utilities/internal/visitor.h" +#include "config_utilities/internal/yaml_utils.h" + +namespace config { + +/** + * @brief Loads a config based on collated YAML data specified via the command line + * + * See fromYaml() for more specific behavioral information. + * + * @tparam ConfigT The config type. This can also be a VirtualConfig or a std::vector. + * @param argc Number of arguments. + * @param argv Actual command line arguments. + * @param name_space Optional namespace to use under the resolved YAML parameter tree. + * @returns The config. + */ +template +ConfigT fromCLI(int argc, char* argv[], const std::string& name_space = "") { + // when parsing CLI locally we don't want to modify the arguments ever + const auto node = internal::loadFromArguments(argc, argv, false); + + ConfigT config; + internal::Visitor::setValues(config, internal::lookupNamespace(node, name_space), true); + return config; +} + +/** + * @brief Create a derived type object based on collated YAML data specified via the command line + * + * See createFromYaml() for more specific behavioral information. + * + * @tparam BaseT Type of the base class to be constructed. + * @tparam Args Other constructor arguments. + * @param argc Number of arguments. + * @param argv Actual command line arguments. + * @param args Other constructor arguments. + * @returns Unique pointer of type base that contains the derived object. + */ +template +std::unique_ptr createFromCLI(int argc, char* argv[], ConstructorArguments... args) { + // when parsing CLI locally we don't want to modify the arguments ever + const auto node = internal::loadFromArguments(argc, argv, false); + return internal::ObjectWithConfigFactory::create(node, args...); +} + +/** + * @brief Create a derived type object based on collated YAML data specified via the command line + * + * See createFromYamlWithNamespace() for more specific behavioral information. + * + * @tparam BaseT Type of the base class to be constructed. + * @tparam Args Other constructor arguments. + * @param argc Number of arguments. + * @param argv Actual command line arguments. + * @param name_space Optionally specify a name space to create the object from. + * @param args Other constructor arguments. + * @returns Unique pointer of type base that contains the derived object. + */ +template +std::unique_ptr createFromYamlWithNamespace(int argc, + char* argv[], + const std::string& name_space, + ConstructorArguments... args) { + // when parsing CLI locally we don't want to modify the arguments ever + const auto node = internal::loadFromArguments(argc, argv, false); + const auto ns_node = internal::lookupNamespace(node, name_space); + return internal::ObjectWithConfigFactory::create(ns_node, args...); +} + +} // namespace config diff --git a/config_utilities/include/config_utilities/parsing/ros2.h b/config_utilities/include/config_utilities/parsing/ros2.h deleted file mode 100644 index 3f820f6..0000000 --- a/config_utilities/include/config_utilities/parsing/ros2.h +++ /dev/null @@ -1,141 +0,0 @@ -/** ----------------------------------------------------------------------------- - * Copyright (c) 2023 Massachusetts Institute of Technology. - * All Rights Reserved. - * - * AUTHORS: Lukas Schmid , Nathan Hughes - * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology - * YEAR: 2023 - * LICENSE: BSD 3-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * -------------------------------------------------------------------------- */ - -#pragma once - -#include -#include -#include -#include - -#include - -#include "config_utilities/factory.h" -#include "config_utilities/internal/string_utils.h" -#include "config_utilities/internal/visitor.h" -#include "config_utilities/internal/yaml_utils.h" -#include "config_utilities/parsing/yaml.h" - -namespace config { - -namespace internal { - -// Translate the parameter server to yaml code. This is inlined to shift the dependency on ROS to the downstream cpp -// file. -inline YAML::Node xmlRpcToYaml(const XmlRpc::XmlRpcValue& xml) { - switch (xml.getType()) { - case XmlRpc::XmlRpcValue::Type::TypeBoolean: - return YAML::Node(static_cast(xml)); - case XmlRpc::XmlRpcValue::Type::TypeInt: - return YAML::Node(static_cast(xml)); - case XmlRpc::XmlRpcValue::Type::TypeDouble: - return YAML::Node(static_cast(xml)); - case XmlRpc::XmlRpcValue::Type::TypeString: - return YAML::Node(static_cast(xml)); - case XmlRpc::XmlRpcValue::Type::TypeArray: { - YAML::Node node(YAML::NodeType::Sequence); - for (int i = 0; i < xml.size(); ++i) { - node.push_back(xmlRpcToYaml(xml[i])); - } - return node; - } - case XmlRpc::XmlRpcValue::Type::TypeStruct: { - YAML::Node node(YAML::NodeType::Map); - for (auto it = xml.begin(); it != xml.end(); ++it) { - node[it->first] = xmlRpcToYaml(it->second); - } - return node; - } - default: - return YAML::Node(); - } -} - -inline YAML::Node rosToYaml(const rclcpp::Node& node) { - YAML::Node root; - - auto all_params = node.list_parameters({}, rcl_interfaces::srv::ListParameters::Request::DEPTH_RECURSIVE); - - for (std::string& name : all_params.names) { - // TODO(nathan) optional namespace filter - - name = name.erase(0, nh.getNamespace().length()); // Remove the nodehandle's namespace. - std::vector name_parts = splitNamespace(name); - std::string local_name = ""; - if (!name_parts.empty()) { - name = name.substr(1); // Remove the leading slash. - local_name = name_parts.back(); - name_parts.pop_back(); - } - - const auto sub_namespace = joinNamespace(name_parts); - - // Get the Xml Value - XmlRpc::XmlRpcValue value; - nh.getParam(name, value); - - // Convert data to yaml. - YAML::Node local_node; - if (local_name.empty()) { - local_node = xmlRpcToYaml(value); - } else { - local_node[local_name] = xmlRpcToYaml(value); - } - moveDownNamespace(local_node, sub_namespace); - mergeYamlNodes(root, local_node); - } - - return root; -} - -} // namespace internal - -/** - * @brief Loads a config from a yaml node. - * - * @tparam ConfigT The config type. This can also be a VirtualConfig or a std::vector. - * @param nh The ROS nodehandle to create the config from. - * @param name_space Optionally specify a name space to create the config from. Separate names with slashes '/'. - * Example: "my_config/my_sub_config". - * @returns The config. - */ -template -ConfigT fromRos(const ros::NodeHandle& nh, const std::string& name_space = "") { - const ros::NodeHandle ns_nh = ros::NodeHandle(nh, name_space); - const YAML::Node node = internal::rosToYaml(ns_nh); - return internal::fromYamlImpl(node, "", static_cast(nullptr)); -} - -} // namespace config diff --git a/config_utilities/src/commandline_utils.cpp b/config_utilities/src/commandline_utils.cpp new file mode 100644 index 0000000..ba7477b --- /dev/null +++ b/config_utilities/src/commandline_utils.cpp @@ -0,0 +1,45 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/internal/commandline_utils.h" + +namespace config::internal { + +YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args) { + YAML::Node node; + return node; +} + +} // namespace config::internal From c8aaf9b416f1c83e85fc426da57e23aa1cac3b9c Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Fri, 3 Jan 2025 14:54:56 +0000 Subject: [PATCH 10/28] remove rclcpp and add context command line function --- config_utilities/CMakeLists.txt | 8 +--- .../config_utilities/parsing/commandline.h | 10 +++- config_utilities/package.xml | 1 + config_utilities/src/commandline.cpp | 47 +++++++++++++++++++ 4 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 config_utilities/src/commandline.cpp diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index 3d5c5c4..7708645 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -11,7 +11,6 @@ include(OptionalPackage) option(BUILD_SHARED_LIBS "Build shared libs" ON) option(CONFIG_UTILS_ENABLE_ROS "Export roscpp and build related code" ON) -option(CONFIG_UTILS_ENABLE_ROS2 "Export rclcpp and build related code" ON) option(CONFIG_UTILS_ENABLE_EIGEN "Export Eigen and build related code" ON) option(CONFIG_UTILS_ENABLE_GLOG "Export glog and build related code" ON) option(CONFIG_UTILS_BUILD_TESTS "Build unit tests" ON) @@ -20,13 +19,13 @@ option(CONFIG_UTILS_BUILD_DEMOS "Build demo executables" ON) find_package(yaml-cpp REQUIRED) find_package(Boost REQUIRED COMPONENTS filesystem system) find_optional(roscpp CONFIG_UTILS_ENABLE_ROS) -find_optional(rclcpp CONFIG_UTILS_ENABLE_ROS2) find_optional(Eigen3 CONFIG_UTILS_ENABLE_EIGEN) find_optional_pkgcfg(libglog CONFIG_UTILS_ENABLE_GLOG) add_library( ${PROJECT_NAME} src/asl_formatter.cpp + src/commandline.cpp src/commandline_utils.cpp src/context.cpp src/conversions.cpp @@ -60,10 +59,6 @@ if(ENABLE_roscpp) target_include_directories(${PROJECT_NAME} PUBLIC ${roscpp_INCLUDE_DIRS}) endif() -if(ENABLE_rclcpp) - target_link_libraries(${PROJECT_NAME} PUBLIC rclcpp::rclcpp) -endif() - if(ENABLE_Eigen3) target_link_libraries(${PROJECT_NAME} PUBLIC Eigen3::Eigen) endif() @@ -74,7 +69,6 @@ endif() message(STATUS "Eigen features enabled: ${ENABLE_Eigen3}") message(STATUS "ROS features enabled: ${ENABLE_roscpp}") -message(STATUS "ROS2 features enabled: ${ENABLE_rclcpp}") message(STATUS "glog features enabled: ${ENABLE_libglog}") if(CONFIG_UTILS_BUILD_DEMOS) diff --git a/config_utilities/include/config_utilities/parsing/commandline.h b/config_utilities/include/config_utilities/parsing/commandline.h index a2c7235..017d1e4 100644 --- a/config_utilities/include/config_utilities/parsing/commandline.h +++ b/config_utilities/include/config_utilities/parsing/commandline.h @@ -35,10 +35,8 @@ #pragma once -#include #include #include -#include #include "config_utilities/factory.h" #include "config_utilities/internal/commandline_utils.h" @@ -47,6 +45,14 @@ namespace config { +/** + * @brief Initialize global config context from the command line + * @param argc Number of arguments. + * @param argv Actual command line arguments. + * @param remove_arguments Remove parsed command line arguments. + */ +void initContext(int argc, char* argv[], bool remove_arguments = true); + /** * @brief Loads a config based on collated YAML data specified via the command line * diff --git a/config_utilities/package.xml b/config_utilities/package.xml index 58039b8..26ee33f 100644 --- a/config_utilities/package.xml +++ b/config_utilities/package.xml @@ -13,6 +13,7 @@ BSD-3-Clause cmake + yaml-cpp cmake diff --git a/config_utilities/src/commandline.cpp b/config_utilities/src/commandline.cpp new file mode 100644 index 0000000..0d58757 --- /dev/null +++ b/config_utilities/src/commandline.cpp @@ -0,0 +1,47 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/parsing/commandline.h" + +#include "config_utilities/internal/context.h" + +namespace config { + +void initContext(int argc, char* argv[], bool remove_arguments) { + const auto node = internal::loadFromArguments(argc, argv, remove_arguments); + internal::Context::update(node, ""); +} + +} // namespace config From 02428005d4b1c5793faad6ea7c81295818fe9692 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Fri, 3 Jan 2025 14:55:34 +0000 Subject: [PATCH 11/28] fix linting warnings and add more coverage for external registry --- .../config_utilities/internal/checks.h | 11 ++---- config_utilities/src/external_registry.cpp | 2 + .../test/tests/external_registry.cpp | 39 ++++++++++++++++++- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/config_utilities/include/config_utilities/internal/checks.h b/config_utilities/include/config_utilities/internal/checks.h index 846aad0..e234e4e 100644 --- a/config_utilities/include/config_utilities/internal/checks.h +++ b/config_utilities/include/config_utilities/internal/checks.h @@ -35,6 +35,7 @@ #pragma once +#include #include #include #include @@ -82,7 +83,7 @@ struct CompareMessageTrait { template class BinaryCheck : public CheckBase { public: - BinaryCheck(const T& param, const T& value, const std::string name = "") + BinaryCheck(const T& param, const T& value, const std::string& name = "") : param_(param), value_(value), name_(name) {} bool valid() const override { return Compare{}(param_, value_); } @@ -186,12 +187,8 @@ class CheckIsOneOf : public CheckBase { : param_(param), candidates_(candidates), name_(name) {} bool valid() const override { - for (const T& cadidate : candidates_) { - if (param_ == cadidate) { - return true; - } - } - return false; + // check that param matches any candidate + return std::any_of(candidates_.begin(), candidates_.end(), [this](const auto& c) { return c == param_; }); } std::string message() const override { diff --git a/config_utilities/src/external_registry.cpp b/config_utilities/src/external_registry.cpp index 583bce2..a686dbd 100644 --- a/config_utilities/src/external_registry.cpp +++ b/config_utilities/src/external_registry.cpp @@ -98,10 +98,12 @@ LibraryGuard::operator bool() const { return !libraries_.empty(); } ExternalRegistry::~ExternalRegistry() { std::vector libraries; for (const auto& [path, lib] : libraries_) { + // NOTE(nathan) technically unreachable (library guards call unload first) libraries.push_back(path); } for (const auto& path : libraries) { + // NOTE(nathan) technically unreachable (library guards call unload first) unload(path); } } diff --git a/config_utilities/test/tests/external_registry.cpp b/config_utilities/test/tests/external_registry.cpp index 2274463..cfa4890 100644 --- a/config_utilities/test/tests/external_registry.cpp +++ b/config_utilities/test/tests/external_registry.cpp @@ -38,6 +38,7 @@ #include #include "config_utilities/logging/log_to_stdout.h" +#include "config_utilities/settings.h" #include "config_utilities/test/external_types.h" #include "config_utilities/test/utils.h" @@ -49,6 +50,27 @@ struct LoggerGuard { std::shared_ptr logger; }; +struct SettingsGuard { + SettingsGuard() {} + ~SettingsGuard() { Settings().restoreDefaults(); } +}; + +TEST(ExternalRegistry, MoveableGuard) { + // guard is valid with at least one library + internal::LibraryGuard guard("some_library_path"); + EXPECT_TRUE(guard); + + // default constructor is invalid state + internal::LibraryGuard other = std::move(guard); + EXPECT_FALSE(guard); + EXPECT_TRUE(other); + + // move assignment should revert invalid/valid states + guard = std::move(other); + EXPECT_FALSE(other); + EXPECT_TRUE(guard); +} + TEST(ExternalRegistry, InstanceLifetimes) { auto plugin_lib = loadExternalFactories("./test_config_utilities_plugins"); auto unmanaged_logger = create("external_logger"); @@ -89,13 +111,25 @@ TEST(ExternalRegistry, InvalidFile) { EXPECT_EQ(logger_guard.logger->lastMessage().find("Unable to load library"), 0); } +TEST(ExternalRegistry, DisableLoading) { + config::test::SettingsGuard settings_guard; + config::Settings().allow_external_libraries = false; + + const auto guard = config::loadExternalFactories("./test_config_utilities_plugins"); + EXPECT_FALSE(guard); +} + } // namespace config::test // globally namespaced to check example compilation TEST(ExternalRegistry, ManagedInstance) { + config::test::SettingsGuard settings_guard; + config::Settings().print_external_allocations = true; + config::ManagedInstance talker; { // scope where external library is loaded - const auto guard = config::loadExternalFactories("./test_config_utilities_plugins"); + const std::vector to_load{"./test_config_utilities_plugins"}; + const auto guard = config::loadExternalFactories(to_load); talker = config::createManaged(config::create("external")); const auto view = talker.get(); ASSERT_TRUE(view); @@ -106,7 +140,8 @@ TEST(ExternalRegistry, ManagedInstance) { EXPECT_FALSE(view); { // scope where external library is loaded - const auto guard = config::loadExternalFactories("./test_config_utilities_plugins"); + const std::vector to_load{"./test_config_utilities_plugins"}; + const auto guard = config::loadExternalFactories(to_load); talker = config::createManaged(config::create("internal")); EXPECT_TRUE(talker); } // external library is unloaded after this point From ebda7b3fe1085056bc8c680e3f520913d9ced3e9 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Fri, 3 Jan 2025 15:07:51 +0000 Subject: [PATCH 12/28] split parsing and utils tests --- config_utilities/test/CMakeLists.txt | 3 +- config_utilities/test/tests/yaml_parsing.cpp | 58 ----------- config_utilities/test/tests/yaml_utils.cpp | 103 +++++++++++++++++++ 3 files changed, 105 insertions(+), 59 deletions(-) create mode 100644 config_utilities/test/tests/yaml_utils.cpp diff --git a/config_utilities/test/CMakeLists.txt b/config_utilities/test/CMakeLists.txt index fc09fbd..3f638dd 100644 --- a/config_utilities/test/CMakeLists.txt +++ b/config_utilities/test/CMakeLists.txt @@ -29,7 +29,8 @@ add_executable( tests/update.cpp tests/validity_checks.cpp tests/virtual_config.cpp - tests/yaml_parsing.cpp) + tests/yaml_parsing.cpp + tests/yaml_utils.cpp) target_include_directories(test_${PROJECT_NAME} PRIVATE include) target_link_libraries(test_${PROJECT_NAME} PRIVATE ${PROJECT_NAME} GTest::gtest_main) diff --git a/config_utilities/test/tests/yaml_parsing.cpp b/config_utilities/test/tests/yaml_parsing.cpp index c28657e..21be46e 100644 --- a/config_utilities/test/tests/yaml_parsing.cpp +++ b/config_utilities/test/tests/yaml_parsing.cpp @@ -38,70 +38,12 @@ #include #include "config_utilities/config.h" -#include "config_utilities/internal/yaml_utils.h" #include "config_utilities/parsing/yaml.h" #include "config_utilities/test/default_config.h" #include "config_utilities/test/utils.h" namespace config::test { -YAML::Node createData() { - YAML::Node data; - data["a"]["b"]["c"] = 1; - data["a"]["b"]["d"] = "test"; - data["a"]["b"]["e"] = std::vector({1, 2, 3}); - data["a"]["b"]["f"] = std::map({{"1_str", 1}, {"2_str", 2}}); - data["a"]["g"] = 3; - return data; -} - -TEST(YamlParsing, lookupNamespace) { - YAML::Node data = createData(); - - expectEqual(data, data); - - YAML::Node data_1 = YAML::Clone(data); - data_1["a"]["b"]["c"] = 2; - EXPECT_FALSE(internal::isEqual(data, data_1)); - - YAML::Node data_2 = internal::lookupNamespace(data, ""); - // NOTE(lschmid): lookupNamespace returns a pointer, so this should be identity. - EXPECT_TRUE(data == data_2); - expectEqual(data, data_2); - - YAML::Node b = internal::lookupNamespace(data, "a/b"); - // NOTE(lschmid): lookupNamespace returns a pointer, so this should be identity. - EXPECT_TRUE(b == data["a"]["b"]); - expectEqual(b, data["a"]["b"]); - - YAML::Node b2 = internal::lookupNamespace(YAML::Clone(data), "a/b"); - - expectEqual(b2, data["a"]["b"]); - - YAML::Node c = internal::lookupNamespace(data, "a/b/c"); - EXPECT_TRUE(c.IsScalar()); - EXPECT_EQ(c.as(), 1); - - YAML::Node invalid = internal::lookupNamespace(data, "a/b/c/d"); - EXPECT_FALSE(invalid.IsDefined()); - EXPECT_FALSE(static_cast(invalid)); - - // Make sure the input node is not modified. - expectEqual(data, createData()); -} - -TEST(YamlParsing, moveDownNamespace) { - YAML::Node data = createData(); - - internal::moveDownNamespace(data, ""); - expectEqual(data, createData()); - - YAML::Node expected_data; - expected_data["a"]["b"]["c"] = createData(); - internal::moveDownNamespace(data, "a/b/c"); - expectEqual(data, expected_data); -} - TEST(YamlParsing, parsefromYaml) { DefaultConfig config; YAML::Node data = DefaultConfig::modifiedValues(); diff --git a/config_utilities/test/tests/yaml_utils.cpp b/config_utilities/test/tests/yaml_utils.cpp new file mode 100644 index 0000000..1b2cff0 --- /dev/null +++ b/config_utilities/test/tests/yaml_utils.cpp @@ -0,0 +1,103 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/internal/yaml_utils.h" + +#include + +#include + +#include "config_utilities/test/utils.h" + +namespace config::test { + +YAML::Node createData() { + YAML::Node data; + data["a"]["b"]["c"] = 1; + data["a"]["b"]["d"] = "test"; + data["a"]["b"]["e"] = std::vector({1, 2, 3}); + data["a"]["b"]["f"] = std::map({{"1_str", 1}, {"2_str", 2}}); + data["a"]["g"] = 3; + return data; +} + +TEST(YamlUtils, lookupNamespace) { + YAML::Node data = createData(); + + expectEqual(data, data); + + YAML::Node data_1 = YAML::Clone(data); + data_1["a"]["b"]["c"] = 2; + EXPECT_FALSE(internal::isEqual(data, data_1)); + + YAML::Node data_2 = internal::lookupNamespace(data, ""); + // NOTE(lschmid): lookupNamespace returns a pointer, so this should be identity. + EXPECT_TRUE(data == data_2); + expectEqual(data, data_2); + + YAML::Node b = internal::lookupNamespace(data, "a/b"); + // NOTE(lschmid): lookupNamespace returns a pointer, so this should be identity. + EXPECT_TRUE(b == data["a"]["b"]); + expectEqual(b, data["a"]["b"]); + + YAML::Node b2 = internal::lookupNamespace(YAML::Clone(data), "a/b"); + + expectEqual(b2, data["a"]["b"]); + + YAML::Node c = internal::lookupNamespace(data, "a/b/c"); + EXPECT_TRUE(c.IsScalar()); + EXPECT_EQ(c.as(), 1); + + YAML::Node invalid = internal::lookupNamespace(data, "a/b/c/d"); + EXPECT_FALSE(invalid.IsDefined()); + EXPECT_FALSE(static_cast(invalid)); + + // Make sure the input node is not modified. + expectEqual(data, createData()); +} + +TEST(YamlUtils, moveDownNamespace) { + YAML::Node data = createData(); + + internal::moveDownNamespace(data, ""); + expectEqual(data, createData()); + + YAML::Node expected_data; + expected_data["a"]["b"]["c"] = createData(); + internal::moveDownNamespace(data, "a/b/c"); + expectEqual(data, expected_data); +} + +} // namespace config::test From 189f53b8d5ad86b9eb464d8faa43b40d875e3a7a Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Tue, 7 Jan 2025 22:47:57 +0000 Subject: [PATCH 13/28] forward extend_sequence arg and add merge tests --- config_utilities/CMakeLists.txt | 1 + config_utilities/src/yaml_utils.cpp | 2 +- config_utilities/test/tests/yaml_utils.cpp | 48 +++++++++++++++++++++- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index 7708645..0dc83df 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -76,6 +76,7 @@ if(CONFIG_UTILS_BUILD_DEMOS) endif() if(CONFIG_UTILS_BUILD_TESTS) + include(CTest) enable_testing() add_subdirectory(test) endif() diff --git a/config_utilities/src/yaml_utils.cpp b/config_utilities/src/yaml_utils.cpp index 1c5f04c..c8f778e 100644 --- a/config_utilities/src/yaml_utils.cpp +++ b/config_utilities/src/yaml_utils.cpp @@ -78,7 +78,7 @@ void mergeYamlNodes(YAML::Node& a, const YAML::Node& b, bool extend_sequences) { if (a[key]) { // Node exists. Merge recursively. YAML::Node a_sub = a[key]; // This node is a ref. - mergeYamlNodes(a_sub, node.second); + mergeYamlNodes(a_sub, node.second, extend_sequences); } else { // Leaf of a, but b continues: insert b a[key] = YAML::Clone(node.second); diff --git a/config_utilities/test/tests/yaml_utils.cpp b/config_utilities/test/tests/yaml_utils.cpp index 1b2cff0..485a1ba 100644 --- a/config_utilities/test/tests/yaml_utils.cpp +++ b/config_utilities/test/tests/yaml_utils.cpp @@ -42,8 +42,8 @@ #include "config_utilities/test/utils.h" namespace config::test { - -YAML::Node createData() { +namespace { +inline YAML::Node createData() { YAML::Node data; data["a"]["b"]["c"] = 1; data["a"]["b"]["d"] = "test"; @@ -53,6 +53,50 @@ YAML::Node createData() { return data; } +inline YAML::Node doMerge(const YAML::Node& lhs, const YAML::Node& rhs, bool extend_lists = false) { + auto result = YAML::Clone(lhs); + internal::mergeYamlNodes(result, rhs, extend_lists); + return result; +} + +} // namespace + +TEST(YamlUtils, mergeYamlNodes) { + auto node_a = YAML::Load(R"""(root: {a: 1, b: 2})"""); + const auto node_b = YAML::Load(R"""(root: {a: 1, c: 3})"""); + internal::mergeYamlNodes(node_a, node_b); + + const auto result = doMerge(node_a, node_b); + const auto expected = YAML::Load(R"""(root: {a: 1, b: 2, c: 3})"""); + expectEqual(result, expected); +} + +TEST(YamlUtils, mergeYamlNodesInvalidKey) { + const auto node_a = YAML::Load(R"""(root: {a: 1, b: 2})"""); + const auto node_b = YAML::Load(R"""(? [root, other]: {a: 1, c: 3})"""); + + const auto result = doMerge(node_a, node_b); + const auto expected = YAML::Load(R"""(root: {a: 1, b: 2})"""); + expectEqual(result, expected); +} + +TEST(YamlUtils, mergeYamlNodesExtend) { + const auto node_a = YAML::Load(R"""(root: {a: 1, b: [2]})"""); + const auto node_b = YAML::Load(R"""(root: {a: 1, b: [4], c: 3})"""); + + { // without extend, lists should override + auto result = doMerge(node_a, node_b); + const auto expected = YAML::Load(R"""(root: {a: 1, b: [4], c: 3})"""); + expectEqual(result, expected); + } + + { // with extend, lists should append + auto result = doMerge(node_a, node_b, true); + const auto expected = YAML::Load(R"""(root: {a: 1, b: [2, 4], c: 3})"""); + expectEqual(result, expected); + } +} + TEST(YamlUtils, lookupNamespace) { YAML::Node data = createData(); From 018e47b574b32665907840da81d363fb73fcb5be Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Wed, 8 Jan 2025 18:20:07 +0000 Subject: [PATCH 14/28] add additional yaml parsing test coverage --- config_utilities/src/yaml_utils.cpp | 3 +- config_utilities/test/tests/yaml_parsing.cpp | 20 ++++++++++ config_utilities/test/tests/yaml_utils.cpp | 40 ++++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/config_utilities/src/yaml_utils.cpp b/config_utilities/src/yaml_utils.cpp index c8f778e..1e97e18 100644 --- a/config_utilities/src/yaml_utils.cpp +++ b/config_utilities/src/yaml_utils.cpp @@ -111,6 +111,7 @@ bool isEqual(const YAML::Node& a, const YAML::Node& b) { if (a.Type() != b.Type()) { return false; } + switch (a.Type()) { case YAML::NodeType::Scalar: return a.Scalar() == b.Scalar(); @@ -139,10 +140,10 @@ bool isEqual(const YAML::Node& a, const YAML::Node& b) { } return true; case YAML::NodeType::Null: - return true; case YAML::NodeType::Undefined: return true; } + return false; } diff --git a/config_utilities/test/tests/yaml_parsing.cpp b/config_utilities/test/tests/yaml_parsing.cpp index 21be46e..fad936f 100644 --- a/config_utilities/test/tests/yaml_parsing.cpp +++ b/config_utilities/test/tests/yaml_parsing.cpp @@ -120,6 +120,26 @@ my_enum: "D" EXPECT_EQ(errors[4]->message(), "Name 'D' is out of bounds for enum with names ['A', 'B', 'C']"); } +TEST(YamlParsing, overflowConversionFailure) { + const auto node = YAML::Load(R"yaml({under: -1, over: 256})yaml"); + + { // values below [0, 255] cause errors + uint8_t value = 0; + std::string error; + EXPECT_FALSE(internal::YamlParser::fromYaml(node, "under", value, "", error)); + EXPECT_EQ(value, 0u); + EXPECT_EQ(error, "Value '-1' overflows storage min of '0'."); + } + + { // values above [0, 255] cause errors + uint8_t value = 0; + std::string error; + EXPECT_FALSE(internal::YamlParser::fromYaml(node, "over", value, "", error)); + EXPECT_EQ(value, 0u); + EXPECT_EQ(error, "Value '256' overflows storage max of '255'."); + } +} + TEST(YamlParsing, setValues) { YAML::Node data = DefaultConfig::modifiedValues(); DefaultConfig config; diff --git a/config_utilities/test/tests/yaml_utils.cpp b/config_utilities/test/tests/yaml_utils.cpp index 485a1ba..6ffa197 100644 --- a/config_utilities/test/tests/yaml_utils.cpp +++ b/config_utilities/test/tests/yaml_utils.cpp @@ -97,6 +97,46 @@ TEST(YamlUtils, mergeYamlNodesExtend) { } } +TEST(YamlUtils, isEqual) { + { // different node types are inequal + const auto node_a = YAML::Load(R"""(5)"""); + const auto node_b = YAML::Load(R"""([5])"""); + EXPECT_TRUE(internal::isEqual(node_a, YAML::Clone(node_a))); + EXPECT_FALSE(internal::isEqual(node_a, node_b)); + } + + { // sequences work as expected + const auto node_a = YAML::Load(R"""([1, 2, 3, 4, 5])"""); + const auto node_b = YAML::Load(R"""([1, 2, 2, 4, 5])"""); + const auto node_c = YAML::Load(R"""([1, 2, 2, 4])"""); + EXPECT_TRUE(internal::isEqual(node_a, YAML::Clone(node_a))); + EXPECT_FALSE(internal::isEqual(node_a, node_b)); + EXPECT_FALSE(internal::isEqual(node_a, node_c)); + EXPECT_FALSE(internal::isEqual(node_b, node_c)); + } + + { // maps work as expected + const auto node_a = YAML::Load(R"""({a: 1, b: 2, c: 3})"""); + const auto node_b = YAML::Load(R"""({a: 1, b: 1, c: 3})"""); + const auto node_c = YAML::Load(R"""({a: 1, d: 1, c: 3})"""); + const auto node_d = YAML::Load(R"""({a: 1, b: 2})"""); + EXPECT_TRUE(internal::isEqual(node_a, YAML::Clone(node_a))); + EXPECT_FALSE(internal::isEqual(node_a, node_b)); + EXPECT_FALSE(internal::isEqual(node_b, node_c)); + EXPECT_FALSE(internal::isEqual(node_a, node_c)); + EXPECT_FALSE(internal::isEqual(node_b, node_d)); + } + + { // null nodes are equal + const auto node_a = YAML::Node(); + const auto node_b = YAML::Node(); + const auto node_c = YAML::Load(R"""({a: 1, d: 1, c: 3})"""); + EXPECT_TRUE(internal::isEqual(node_a, node_b)); + EXPECT_FALSE(internal::isEqual(node_a, node_c)); + EXPECT_FALSE(internal::isEqual(node_b, node_c)); + } +} + TEST(YamlUtils, lookupNamespace) { YAML::Node data = createData(); From 921f14f00de33a208984532058a42e0094de5b84 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Wed, 8 Jan 2025 18:32:25 +0000 Subject: [PATCH 15/28] fix broken test and make merge test more complicated --- config_utilities/test/tests/yaml_parsing.cpp | 2 +- config_utilities/test/tests/yaml_utils.cpp | 43 ++++++++++++++++++-- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/config_utilities/test/tests/yaml_parsing.cpp b/config_utilities/test/tests/yaml_parsing.cpp index fad936f..b1a1199 100644 --- a/config_utilities/test/tests/yaml_parsing.cpp +++ b/config_utilities/test/tests/yaml_parsing.cpp @@ -128,7 +128,7 @@ TEST(YamlParsing, overflowConversionFailure) { std::string error; EXPECT_FALSE(internal::YamlParser::fromYaml(node, "under", value, "", error)); EXPECT_EQ(value, 0u); - EXPECT_EQ(error, "Value '-1' overflows storage min of '0'."); + EXPECT_EQ(error, "Value '-1' underflows storage min of '0'."); } { // values above [0, 255] cause errors diff --git a/config_utilities/test/tests/yaml_utils.cpp b/config_utilities/test/tests/yaml_utils.cpp index 6ffa197..e80502c 100644 --- a/config_utilities/test/tests/yaml_utils.cpp +++ b/config_utilities/test/tests/yaml_utils.cpp @@ -81,18 +81,53 @@ TEST(YamlUtils, mergeYamlNodesInvalidKey) { } TEST(YamlUtils, mergeYamlNodesExtend) { - const auto node_a = YAML::Load(R"""(root: {a: 1, b: [2]})"""); - const auto node_b = YAML::Load(R"""(root: {a: 1, b: [4], c: 3})"""); + // NOTE(nathan) structured to require appending at different levels of recursion + const auto node_a = YAML::Load(R"""( +root: + a: 1 + b: [2] + c: + foo: [1, 2, 3] + bar: [{1: 2, 3: 4}, {5, 6}] +other: [1, 2] +)"""); + const auto node_b = YAML::Load(R"""( +root: + a: 1 + b: [4] + c: + bar: [7] + d: 3.0 +other: [3, 4, 5] +)"""); { // without extend, lists should override auto result = doMerge(node_a, node_b); - const auto expected = YAML::Load(R"""(root: {a: 1, b: [4], c: 3})"""); + const auto expected = YAML::Load(R"""( +root: + a: 1 + b: [4] + c: + foo: [1, 2, 3] + bar: [7] + d: 3.0 +other: [3, 4, 5] +)"""); expectEqual(result, expected); } { // with extend, lists should append auto result = doMerge(node_a, node_b, true); - const auto expected = YAML::Load(R"""(root: {a: 1, b: [2, 4], c: 3})"""); + const auto expected = YAML::Load(R"""( +root: + a: 1 + b: [2, 4] + c: + foo: [1, 2, 3] + bar: [{1: 2, 3: 4}, {5, 6}, 7] + d: 3.0 +other: [1, 2, 3, 4, 5] +)"""); expectEqual(result, expected); } } From 0e05454b06ecbc58602123f3e7439b90b1d93399 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Thu, 9 Jan 2025 02:33:59 +0000 Subject: [PATCH 16/28] (wip) initial implementation of parsing --- config_utilities/CMakeLists.txt | 5 ++- config_utilities/src/commandline_utils.cpp | 47 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index 0dc83df..5f1e30c 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -17,7 +17,7 @@ option(CONFIG_UTILS_BUILD_TESTS "Build unit tests" ON) option(CONFIG_UTILS_BUILD_DEMOS "Build demo executables" ON) find_package(yaml-cpp REQUIRED) -find_package(Boost REQUIRED COMPONENTS filesystem system) +find_package(Boost REQUIRED COMPONENTS filesystem system program_options) find_optional(roscpp CONFIG_UTILS_ENABLE_ROS) find_optional(Eigen3 CONFIG_UTILS_ENABLE_EIGEN) find_optional_pkgcfg(libglog CONFIG_UTILS_ENABLE_GLOG) @@ -46,7 +46,8 @@ add_library( target_link_libraries( ${PROJECT_NAME} PUBLIC yaml-cpp - PRIVATE Boost::filesystem Boost::system Boost::boost ${CMAKE_DL_LIBS}) + PRIVATE Boost::filesystem Boost::system Boost::program_options Boost::boost + ${CMAKE_DL_LIBS}) target_include_directories( ${PROJECT_NAME} PUBLIC $ $) diff --git a/config_utilities/src/commandline_utils.cpp b/config_utilities/src/commandline_utils.cpp index ba7477b..1208c40 100644 --- a/config_utilities/src/commandline_utils.cpp +++ b/config_utilities/src/commandline_utils.cpp @@ -35,10 +35,57 @@ #include "config_utilities/internal/commandline_utils.h" +#include + +#include + +#include "config_utilities/internal/yaml_utils.h" + namespace config::internal { +namespace po = boost::program_options; + YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args) { YAML::Node node; + + po::options_description desc("config_utilities options"); + // clang-format off + desc.add_options() + ("config-utilities-file", po::value>(), "file(s) to load") + ("config-utilities-yaml", po::value>(), "yaml to load"); + // clang-format on + + po::variables_map vm; + po::store(po::command_line_parser(argc, argv).options(desc).allow_unregistered().run(), vm); + po::notify(vm); + + if (vm.count("config-utilities-file")) { + const auto files = vm["config-utilities-file"].as>(); + for (const auto& file : files) { + YAML::Node file_node; + try { + file_node = YAML::LoadFile(file); + } catch (const std::exception& e) { + std::cerr << "Failure for '" << file << "': " << e.what() << std::endl; + } + internal::mergeYamlNodes(node, file_node, true); + } + } + + if (vm.count("config-utilities-yaml")) { + const auto entries = vm["config-utilities-yaml"].as>(); + for (const auto& entry : entries) { + YAML::Node cli_node; + try { + cli_node = YAML::Load(entry); + } catch (const std::exception& e) { + std::cerr << "Failure for '" << entry << "': " << e.what() << std::endl; + } + + internal::mergeYamlNodes(node, cli_node, true); + } + } + return node; } From 922a9e4a0bf9ad0bda8bc51070f98624ecd7714f Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Thu, 9 Jan 2025 13:27:53 +0000 Subject: [PATCH 17/28] condense files and allow for multiple tokens --- config_utilities/CMakeLists.txt | 1 - .../internal/commandline_utils.h | 49 ---------- .../config_utilities/parsing/commandline.h | 11 ++- config_utilities/src/commandline.cpp | 54 +++++++++++ config_utilities/src/commandline_utils.cpp | 92 ------------------- 5 files changed, 64 insertions(+), 143 deletions(-) delete mode 100644 config_utilities/include/config_utilities/internal/commandline_utils.h delete mode 100644 config_utilities/src/commandline_utils.cpp diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index 5f1e30c..94ba2fe 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -26,7 +26,6 @@ add_library( ${PROJECT_NAME} src/asl_formatter.cpp src/commandline.cpp - src/commandline_utils.cpp src/context.cpp src/conversions.cpp src/external_registry.cpp diff --git a/config_utilities/include/config_utilities/internal/commandline_utils.h b/config_utilities/include/config_utilities/internal/commandline_utils.h deleted file mode 100644 index 216c37f..0000000 --- a/config_utilities/include/config_utilities/internal/commandline_utils.h +++ /dev/null @@ -1,49 +0,0 @@ -/** ----------------------------------------------------------------------------- - * Copyright (c) 2023 Massachusetts Institute of Technology. - * All Rights Reserved. - * - * AUTHORS: Lukas Schmid , Nathan Hughes - * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology - * YEAR: 2023 - * LICENSE: BSD 3-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * -------------------------------------------------------------------------- */ - -#pragma once - -#include - -namespace config::internal { - -/** - * @brief Parse and collate YAML node from arguments, optionally removing arguments - * @param argc Number of command line arguments - * @param argv Command line argument strings - */ -YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args); - -} // namespace config::internal diff --git a/config_utilities/include/config_utilities/parsing/commandline.h b/config_utilities/include/config_utilities/parsing/commandline.h index 017d1e4..bd5c654 100644 --- a/config_utilities/include/config_utilities/parsing/commandline.h +++ b/config_utilities/include/config_utilities/parsing/commandline.h @@ -39,11 +39,20 @@ #include #include "config_utilities/factory.h" -#include "config_utilities/internal/commandline_utils.h" #include "config_utilities/internal/visitor.h" #include "config_utilities/internal/yaml_utils.h" namespace config { +namespace internal { + +/** + * @brief Parse and collate YAML node from arguments, optionally removing arguments + * @param argc Number of command line arguments + * @param argv Command line argument strings + */ +YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args); + +} // namespace internal /** * @brief Initialize global config context from the command line diff --git a/config_utilities/src/commandline.cpp b/config_utilities/src/commandline.cpp index 0d58757..81fda40 100644 --- a/config_utilities/src/commandline.cpp +++ b/config_utilities/src/commandline.cpp @@ -35,9 +35,63 @@ #include "config_utilities/parsing/commandline.h" +#include + +#include + #include "config_utilities/internal/context.h" +#include "config_utilities/internal/yaml_utils.h" namespace config { +namespace internal { + +namespace po = boost::program_options; + +YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args) { + YAML::Node node; + + po::options_description desc("config_utilities options"); + // clang-format off + desc.add_options() + ("config-utilities-file", po::value>(), "file(s) to load") + ("config-utilities-yaml", po::value>()->multitoken(), "yaml to load"); + // clang-format on + + po::variables_map vm; + po::store(po::command_line_parser(argc, argv).options(desc).allow_unregistered().run(), vm); + po::notify(vm); + + if (vm.count("config-utilities-file")) { + const auto files = vm["config-utilities-file"].as>(); + for (const auto& file : files) { + YAML::Node file_node; + try { + file_node = YAML::LoadFile(file); + } catch (const std::exception& e) { + std::cerr << "Failure for '" << file << "': " << e.what() << std::endl; + } + internal::mergeYamlNodes(node, file_node, true); + } + } + + if (vm.count("config-utilities-yaml")) { + const auto entries = vm["config-utilities-yaml"].as>(); + for (const auto& entry : entries) { + YAML::Node cli_node; + try { + cli_node = YAML::Load(entry); + } catch (const std::exception& e) { + std::cerr << "Failure for '" << entry << "': " << e.what() << std::endl; + } + + internal::mergeYamlNodes(node, cli_node, true); + } + } + + return node; +} + +} // namespace internal void initContext(int argc, char* argv[], bool remove_arguments) { const auto node = internal::loadFromArguments(argc, argv, remove_arguments); diff --git a/config_utilities/src/commandline_utils.cpp b/config_utilities/src/commandline_utils.cpp deleted file mode 100644 index 1208c40..0000000 --- a/config_utilities/src/commandline_utils.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/** ----------------------------------------------------------------------------- - * Copyright (c) 2023 Massachusetts Institute of Technology. - * All Rights Reserved. - * - * AUTHORS: Lukas Schmid , Nathan Hughes - * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology - * YEAR: 2023 - * LICENSE: BSD 3-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * -------------------------------------------------------------------------- */ - -#include "config_utilities/internal/commandline_utils.h" - -#include - -#include - -#include "config_utilities/internal/yaml_utils.h" - -namespace config::internal { - -namespace po = boost::program_options; - -YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args) { - YAML::Node node; - - po::options_description desc("config_utilities options"); - // clang-format off - desc.add_options() - ("config-utilities-file", po::value>(), "file(s) to load") - ("config-utilities-yaml", po::value>(), "yaml to load"); - // clang-format on - - po::variables_map vm; - po::store(po::command_line_parser(argc, argv).options(desc).allow_unregistered().run(), vm); - po::notify(vm); - - if (vm.count("config-utilities-file")) { - const auto files = vm["config-utilities-file"].as>(); - for (const auto& file : files) { - YAML::Node file_node; - try { - file_node = YAML::LoadFile(file); - } catch (const std::exception& e) { - std::cerr << "Failure for '" << file << "': " << e.what() << std::endl; - } - internal::mergeYamlNodes(node, file_node, true); - } - } - - if (vm.count("config-utilities-yaml")) { - const auto entries = vm["config-utilities-yaml"].as>(); - for (const auto& entry : entries) { - YAML::Node cli_node; - try { - cli_node = YAML::Load(entry); - } catch (const std::exception& e) { - std::cerr << "Failure for '" << entry << "': " << e.what() << std::endl; - } - - internal::mergeYamlNodes(node, cli_node, true); - } - } - - return node; -} - -} // namespace config::internal From b433de04fe8736454dbd985dc85de39e0f5b0063 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Thu, 9 Jan 2025 17:16:27 +0000 Subject: [PATCH 18/28] implement custom parser because ros can't escape correctly --- config_utilities/CMakeLists.txt | 5 +- .../config_utilities/internal/context.h | 15 ++ .../config_utilities/parsing/commandline.h | 4 +- .../config_utilities/parsing/context.h | 76 +++++++ config_utilities/src/commandline.cpp | 197 ++++++++++++++---- 5 files changed, 257 insertions(+), 40 deletions(-) create mode 100644 config_utilities/include/config_utilities/parsing/context.h diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index 94ba2fe..8f83da3 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -17,7 +17,7 @@ option(CONFIG_UTILS_BUILD_TESTS "Build unit tests" ON) option(CONFIG_UTILS_BUILD_DEMOS "Build demo executables" ON) find_package(yaml-cpp REQUIRED) -find_package(Boost REQUIRED COMPONENTS filesystem system program_options) +find_package(Boost REQUIRED COMPONENTS filesystem system) find_optional(roscpp CONFIG_UTILS_ENABLE_ROS) find_optional(Eigen3 CONFIG_UTILS_ENABLE_EIGEN) find_optional_pkgcfg(libglog CONFIG_UTILS_ENABLE_GLOG) @@ -45,8 +45,7 @@ add_library( target_link_libraries( ${PROJECT_NAME} PUBLIC yaml-cpp - PRIVATE Boost::filesystem Boost::system Boost::program_options Boost::boost - ${CMAKE_DL_LIBS}) + PRIVATE Boost::filesystem Boost::system Boost::boost ${CMAKE_DL_LIBS}) target_include_directories( ${PROJECT_NAME} PUBLIC $ $) diff --git a/config_utilities/include/config_utilities/internal/context.h b/config_utilities/include/config_utilities/internal/context.h index e435eaf..3c8f00c 100644 --- a/config_utilities/include/config_utilities/internal/context.h +++ b/config_utilities/include/config_utilities/internal/context.h @@ -39,6 +39,9 @@ #include +#include "config_utilities/factory.h" +#include "config_utilities/internal/visitor.h" + namespace config::internal { /** @@ -50,6 +53,18 @@ class Context { static void update(const YAML::Node& other, const std::string& ns); + template + static std::unique_ptr create(ConstructorArguments... args) { + return internal::ObjectWithConfigFactory::create(instance().contents_, args...); + } + + template + static ConfigT loadConfig(const std::string& name_space = "") { + ConfigT config; + internal::Visitor::setValues(config, internal::lookupNamespace(instance().contents_, name_space), true); + return config; + } + private: Context() = default; static Context& instance(); diff --git a/config_utilities/include/config_utilities/parsing/commandline.h b/config_utilities/include/config_utilities/parsing/commandline.h index bd5c654..d922620 100644 --- a/config_utilities/include/config_utilities/parsing/commandline.h +++ b/config_utilities/include/config_utilities/parsing/commandline.h @@ -50,7 +50,7 @@ namespace internal { * @param argc Number of command line arguments * @param argv Command line argument strings */ -YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args); +YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args); } // namespace internal @@ -60,7 +60,7 @@ YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args); * @param argv Actual command line arguments. * @param remove_arguments Remove parsed command line arguments. */ -void initContext(int argc, char* argv[], bool remove_arguments = true); +void initContext(int& argc, char* argv[], bool remove_arguments = true); /** * @brief Loads a config based on collated YAML data specified via the command line diff --git a/config_utilities/include/config_utilities/parsing/context.h b/config_utilities/include/config_utilities/parsing/context.h new file mode 100644 index 0000000..8a911fc --- /dev/null +++ b/config_utilities/include/config_utilities/parsing/context.h @@ -0,0 +1,76 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#pragma once + +#include + +#include "config_utilities/internal/context.h" + +namespace config { + +/** + * @brief Loads a config from the global context + * + * @tparam ConfigT The config type. This can also be a VirtualConfig or a std::vector. + * @param name_space Optionally specify a name space to create the config from. Separate names with slashes '/'. + * Example: "my_config/my_sub_config". + * @returns The config. + */ +template +ConfigT fromContext(const std::string& name_space = "") { + return internal::Context::loadConfig(name_space); +} + +/** + * @brief Create a derived type object based on a the data stored in a yaml node. All derived types need to be + * registered to the factory using a static config::Registration struct. They + * need to implement a config as a public member struct named 'Config' and use the config as the first constructor + * argument. + * + * @tparam BaseT Type of the base class to be constructed. + * @tparam Args Other constructor arguments. Note that each unique set of constructor arguments will result in a + * different base-entry in the factory. + * @param args Other constructor arguments. + * @returns Unique pointer of type base that contains the derived object. + */ +template +std::unique_ptr createFromContext(ConstructorArguments... args) { + return internal::Context::create(args...); +} + +// TODO(nathan) pull other options from ROS file + +} // namespace config diff --git a/config_utilities/src/commandline.cpp b/config_utilities/src/commandline.cpp index 81fda40..e8d92af 100644 --- a/config_utilities/src/commandline.cpp +++ b/config_utilities/src/commandline.cpp @@ -35,65 +35,192 @@ #include "config_utilities/parsing/commandline.h" +#include #include -#include - #include "config_utilities/internal/context.h" #include "config_utilities/internal/yaml_utils.h" namespace config { namespace internal { -namespace po = boost::program_options; +namespace fs = std::filesystem; -YAML::Node loadFromArguments(int argc, char* argv[], bool remove_args) { - YAML::Node node; +struct CliParser { + static constexpr auto FILE_OPT = "--config-utilities-file"; + static constexpr auto YAML_OPT = "--config-utilities-yaml"; + + CliParser() = default; + CliParser& parse(int& argc, char* argv[], bool remove_args); + + std::vector files; + std::vector yaml_entries; +}; + +struct Span { + int pos = 0; + int num_tokens = 1; + std::string extractTokens(int argc, char* argv[]) const; +}; + +std::string Span::extractTokens(int argc, char* argv[]) const { + if (pos < 0) { + return ""; + } + + std::stringstream ss; + const auto total = std::min(argc, pos + num_tokens + 1); + for (int i = pos + 1; i < total; ++i) { + ss << argv[i]; + if (i < total - 1) { + ss << " "; + } + } + + return ss.str(); +} + +std::ostream& operator<<(std::ostream& out, const Span& span) { + out << "[" << span.pos << ", " << span.pos + span.num_tokens << "]"; + return out; +} + +std::optional getSpan(int argc, char* argv[], int pos, bool parse_all, std::string& error) { + int index = pos + 1; + while (index < argc) { + const std::string curr_opt = argv[index]; + const bool is_flag = !curr_opt.empty() && curr_opt[0] == '-'; + if (is_flag && index == pos + 1) { + error = parse_all ? "at least one value required!" : "missing required value!"; + return std::nullopt; + } + + if (!parse_all) { + return Span{pos, 1}; + } + + if (is_flag) { + break; // stop parsing the span + } + + ++index; + } + + // return multi-token span + return Span{pos, index - pos - 1}; +} + +void removeSpan(int& argc, char* argv[], const Span& span) { + for (int token = span.num_tokens; token >= 0; --token) { + for (int i = span.pos + token; i < argc - (span.num_tokens - token); ++i) { + // bubble-sort esque shuffle to move args to end + std::swap(argv[i], argv[i + 1]); + } + } + + argc -= span.num_tokens + 1; +} - po::options_description desc("config_utilities options"); - // clang-format off - desc.add_options() - ("config-utilities-file", po::value>(), "file(s) to load") - ("config-utilities-yaml", po::value>()->multitoken(), "yaml to load"); - // clang-format on - - po::variables_map vm; - po::store(po::command_line_parser(argc, argv).options(desc).allow_unregistered().run(), vm); - po::notify(vm); - - if (vm.count("config-utilities-file")) { - const auto files = vm["config-utilities-file"].as>(); - for (const auto& file : files) { - YAML::Node file_node; - try { - file_node = YAML::LoadFile(file); - } catch (const std::exception& e) { - std::cerr << "Failure for '" << file << "': " << e.what() << std::endl; +void removeSpans(int& argc, char* argv[], const std::map>& arg_spans) { + std::vector spans; + for (const auto& [key, key_spans] : arg_spans) { + for (const auto& span : key_spans) { + spans.push_back(span); + } + } + + std::sort(spans.begin(), spans.end(), [](const auto& lhs, const auto& rhs) { return lhs.pos > rhs.pos; }); + for (const auto& span : spans) { + removeSpan(argc, argv, span); + } +} + +CliParser& CliParser::parse(int& argc, char* argv[], bool remove_args) { + std::map> spans; + + int i = 0; + while (i < argc) { + const std::string curr_opt(argv[i]); + std::string error; + std::optional curr_span; + if (curr_opt == FILE_OPT || curr_opt == YAML_OPT) { + curr_span = getSpan(argc, argv, i, curr_opt == YAML_OPT, error); + } + + if (curr_span) { + auto iter = spans.find(curr_opt); + if (iter == spans.end()) { + iter = spans.emplace(curr_opt, std::vector()).first; } - internal::mergeYamlNodes(node, file_node, true); + + iter->second.emplace_back(*curr_span); + i += curr_span->num_tokens; + continue; + } + + if (!error.empty()) { + std::cerr << "Parse issue for '" << curr_opt << "': " << error << std::endl; } + + ++i; } - if (vm.count("config-utilities-yaml")) { - const auto entries = vm["config-utilities-yaml"].as>(); - for (const auto& entry : entries) { - YAML::Node cli_node; - try { - cli_node = YAML::Load(entry); - } catch (const std::exception& e) { - std::cerr << "Failure for '" << entry << "': " << e.what() << std::endl; + for (const auto& [key, key_spans] : spans) { + for (const auto& span : key_spans) { + if (key == FILE_OPT) { + files.push_back(span.extractTokens(argc, argv)); + } else { + yaml_entries.push_back(span.extractTokens(argc, argv)); } + } + } - internal::mergeYamlNodes(node, cli_node, true); + if (remove_args) { + removeSpans(argc, argv, spans); + } + + return *this; +} + +YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) { + YAML::Node node; + + const auto parser = CliParser().parse(argc, argv, remove_args); + + for (const auto& file : parser.files) { + if (!fs::exists(file)) { + // TODO(nathan) log instead of manual print + std::cerr << "File " << file << " does not exist!" << std::endl; } + + YAML::Node file_node; + try { + file_node = YAML::LoadFile(file); + } catch (const std::exception& e) { + std::cerr << "Failure for " << file << ": " << e.what() << std::endl; + } + + internal::mergeYamlNodes(node, file_node, true); + } + + for (const auto& entry : parser.yaml_entries) { + YAML::Node cli_node; + try { + cli_node = YAML::Load(entry); + } catch (const std::exception& e) { + std::cerr << "Failure for '" << entry << "': " << e.what() << std::endl; + } + + internal::mergeYamlNodes(node, cli_node, true); } + std::cerr << "Loaded YAML:\n" << node << std::endl;; return node; } } // namespace internal -void initContext(int argc, char* argv[], bool remove_arguments) { +void initContext(int& argc, char* argv[], bool remove_arguments) { const auto node = internal::loadFromArguments(argc, argv, remove_arguments); internal::Context::update(node, ""); } From f00a2c87355866308260d889a2df78a20bd8bba1 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Thu, 9 Jan 2025 17:18:38 +0000 Subject: [PATCH 19/28] drop debug print --- config_utilities/src/commandline.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config_utilities/src/commandline.cpp b/config_utilities/src/commandline.cpp index e8d92af..8f856e6 100644 --- a/config_utilities/src/commandline.cpp +++ b/config_utilities/src/commandline.cpp @@ -159,6 +159,7 @@ CliParser& CliParser::parse(int& argc, char* argv[], bool remove_args) { } if (!error.empty()) { + // TODO(nathan) log instead of manual print std::cerr << "Parse issue for '" << curr_opt << "': " << error << std::endl; } @@ -197,6 +198,7 @@ YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) { try { file_node = YAML::LoadFile(file); } catch (const std::exception& e) { + // TODO(nathan) log instead of manual print std::cerr << "Failure for " << file << ": " << e.what() << std::endl; } @@ -208,13 +210,13 @@ YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) { try { cli_node = YAML::Load(entry); } catch (const std::exception& e) { + // TODO(nathan) log instead of manual print std::cerr << "Failure for '" << entry << "': " << e.what() << std::endl; } internal::mergeYamlNodes(node, cli_node, true); } - std::cerr << "Loaded YAML:\n" << node << std::endl;; return node; } From 355e6bb4fd58f444d8e9c99b7e9a0eb1b9b56f21 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Thu, 9 Jan 2025 22:12:16 +0000 Subject: [PATCH 20/28] add ability to namespace files --- config_utilities/src/commandline.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/config_utilities/src/commandline.cpp b/config_utilities/src/commandline.cpp index 8f856e6..5df4c2d 100644 --- a/config_utilities/src/commandline.cpp +++ b/config_utilities/src/commandline.cpp @@ -53,7 +53,7 @@ struct CliParser { CliParser() = default; CliParser& parse(int& argc, char* argv[], bool remove_args); - std::vector files; + std::vector files; std::vector yaml_entries; }; @@ -188,10 +188,20 @@ YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) { const auto parser = CliParser().parse(argc, argv, remove_args); - for (const auto& file : parser.files) { + for (const auto& entry : parser.files) { + auto filepath = entry; + std::string ns; + const auto index = filepath.find("@"); + if (index != std::string::npos) { + ns = filepath.substr(index + 1); + filepath = filepath.substr(0, index); + } + + std::filesystem::path file(filepath); if (!fs::exists(file)) { // TODO(nathan) log instead of manual print std::cerr << "File " << file << " does not exist!" << std::endl; + continue; } YAML::Node file_node; @@ -200,6 +210,11 @@ YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) { } catch (const std::exception& e) { // TODO(nathan) log instead of manual print std::cerr << "Failure for " << file << ": " << e.what() << std::endl; + continue; + } + + if (!ns.empty()) { + internal::moveDownNamespace(file_node, ns); } internal::mergeYamlNodes(node, file_node, true); From f00edbc140b5f7e3214e8e8c140e0d9ed3f71b9d Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Fri, 10 Jan 2025 18:32:03 +0000 Subject: [PATCH 21/28] drop launch extension --- config_utilities_launch/.flake8 | 5 - config_utilities_launch/LICENSE | 25 ---- .../config_utilities_launch/__init__.py | 107 ------------------ config_utilities_launch/package.xml | 18 --- .../resource/config_utilities_launch | 0 config_utilities_launch/setup.cfg | 4 - config_utilities_launch/setup.py | 24 ---- .../test/test_copyright.py | 25 ---- config_utilities_launch/test/test_flake8.py | 25 ---- config_utilities_launch/test/test_pep257.py | 23 ---- config_utilities_launch/test_launch.yaml | 13 --- 11 files changed, 269 deletions(-) delete mode 100644 config_utilities_launch/.flake8 delete mode 100644 config_utilities_launch/LICENSE delete mode 100644 config_utilities_launch/config_utilities_launch/__init__.py delete mode 100644 config_utilities_launch/package.xml delete mode 100644 config_utilities_launch/resource/config_utilities_launch delete mode 100644 config_utilities_launch/setup.cfg delete mode 100644 config_utilities_launch/setup.py delete mode 100644 config_utilities_launch/test/test_copyright.py delete mode 100644 config_utilities_launch/test/test_flake8.py delete mode 100644 config_utilities_launch/test/test_pep257.py delete mode 100644 config_utilities_launch/test_launch.yaml diff --git a/config_utilities_launch/.flake8 b/config_utilities_launch/.flake8 deleted file mode 100644 index 5768200..0000000 --- a/config_utilities_launch/.flake8 +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -max-line-length = 88 -extend-ignore = E203 -per-file-ignores = - *__init__.py:F401,F403 diff --git a/config_utilities_launch/LICENSE b/config_utilities_launch/LICENSE deleted file mode 100644 index 574ef07..0000000 --- a/config_utilities_launch/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - * Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/config_utilities_launch/config_utilities_launch/__init__.py b/config_utilities_launch/config_utilities_launch/__init__.py deleted file mode 100644 index 89351a1..0000000 --- a/config_utilities_launch/config_utilities_launch/__init__.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Main entry point for the config_utilities_launch package.""" - -from launch.action import Action -from launch.frontend import expose_action, Entity, Parser -from launch.launch_context import LaunchContext -from launch.utilities import perform_substitutions, normalize_to_list_of_substitutions -from launch_ros.actions import Node - -from typing import Optional, List -from dataclasses import dataclass - - -@dataclass -class ConfigFile: - """Quick class representing a file.""" - - filepath: str # TODO(nathan) this isn't just a str - ns: Optional[str] = None # TODO(nathan) this isn't just a str - allow_substs: bool = False - - -@dataclass -class ConfigField: - """Quick class representing a single config value at a name.""" - - name: str - value: str # TODO(nathan) this isn't just a str - allow_substs: bool = False - - -def _substitute(context, value): - match value: - case ConfigFile(filepath, ns, allow_substs): - if allow_substs: - filepath = perform_substitutions( - context, normalize_to_list_of_substitutions(filepath) - ) - return {ns: filepath} - case ConfigField(name, value, allow_substs): - if allow_substs: - value = perform_substitutions( - context, normalize_to_list_of_substitutions(value) - ) - return {name: value} - - -def _parse_config_section(section, parser): - name_attr = section.get_attr("name", optional=True) - from_attr = section.get_attr("from", optional=True) - allow_attr = section.get_attr("allow_substs", optional=True) - ns_attr = section.get_attr("ns", optional=True) if from_attr is not None else None - value_attr = ( - section.get_attr("value", optional=True) if name_attr is not None else None - ) - - if from_attr is not None and name_attr is not None: - raise RuntimeError( - "Invalid config field: must either be a named param or a file" - ) - - if from_attr is None and name_attr is None: - raise RuntimeError("Invalid config field: must have a 'name' or 'from' tag") - - section.assert_entity_completely_parsed() - # TODO(nathan) validate this parses bools correctly - allow_substs = False if allow_attr is not None else bool(allow_attr) - - if from_attr is not None: - from_attr = parser.parse_substitution(from_attr) - if ns_attr is not None: - ns_attr = parser.parse_substitution(ns_attr) - - return ConfigFile(filepath=from_attr, ns=ns_attr, allow_substs=allow_substs) - else: - value_attr = parser.parse_substitution(value_attr) - return ConfigField(name=name_attr, value=value_attr, allow_substs=allow_substs) - - -def _parse_config_list(config, parser): - return [_parse_config_section(x, parser) for x in config] - - -@expose_action("configured_node") -class ConfiguredNode(Node): - """Extension of Node that allows for collating arbitrary YAML files.""" - - def __init__(self, *args, config=None, **kwargs): - """Construct a node.""" - super().__init__(*args, **kwargs) - self._config = config - - @classmethod - def parse(cls, entity: Entity, parser: Parser): - """Parse a node from a launch file.""" - _, kwargs = super().parse(entity, parser) - config = entity.get_attr("config", optional=True, data_type=List[Entity]) - if config is not None: - kwargs["config"] = _parse_config_list(config, parser) - - return cls, kwargs - - def execute(self, context: LaunchContext) -> Optional[List[Action]]: - """Collate parsed config entries and pass to node.""" - print(self._config) - self._config = [_substitute(context, x) for x in self._config] - print(self._config) - return super().execute(context) diff --git a/config_utilities_launch/package.xml b/config_utilities_launch/package.xml deleted file mode 100644 index 74b4c5d..0000000 --- a/config_utilities_launch/package.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - config_utilities_launch - 0.0.0 - TODO: Package description - nathan - BSD-3-Clause - - ament_copyright - ament_flake8 - ament_pep257 - python3-pytest - - - ament_python - - diff --git a/config_utilities_launch/resource/config_utilities_launch b/config_utilities_launch/resource/config_utilities_launch deleted file mode 100644 index e69de29..0000000 diff --git a/config_utilities_launch/setup.cfg b/config_utilities_launch/setup.cfg deleted file mode 100644 index 9b4b8d3..0000000 --- a/config_utilities_launch/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[develop] -script_dir=$base/lib/config_utilities_launch -[install] -install_scripts=$base/lib/config_utilities_launch diff --git a/config_utilities_launch/setup.py b/config_utilities_launch/setup.py deleted file mode 100644 index 8df0b49..0000000 --- a/config_utilities_launch/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -from setuptools import find_packages, setup - -package_name = "config_utilities_launch" - -setup( - name=package_name, - version="0.0.0", - packages=find_packages(exclude=["test"]), - data_files=[ - ("share/ament_index/resource_index/packages", ["resource/" + package_name]), - ("share/" + package_name, ["package.xml"]), - ], - install_requires=["setuptools"], - maintainer="nathan", - maintainer_email="nathan.h.hughes@gmail.com", - description="TODO: Package description", - license="BSD-3-Clause", - tests_require=["pytest"], - entry_points={ - "launch.frontend.launch_extension": [ - "config_utilities_launch = config_utilities_launch" - ], - }, -) diff --git a/config_utilities_launch/test/test_copyright.py b/config_utilities_launch/test/test_copyright.py deleted file mode 100644 index 97a3919..0000000 --- a/config_utilities_launch/test/test_copyright.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2015 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ament_copyright.main import main -import pytest - - -# Remove the `skip` decorator once the source file(s) have a copyright header -@pytest.mark.skip(reason='No copyright header has been placed in the generated source file.') -@pytest.mark.copyright -@pytest.mark.linter -def test_copyright(): - rc = main(argv=['.', 'test']) - assert rc == 0, 'Found errors' diff --git a/config_utilities_launch/test/test_flake8.py b/config_utilities_launch/test/test_flake8.py deleted file mode 100644 index 27ee107..0000000 --- a/config_utilities_launch/test/test_flake8.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2017 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ament_flake8.main import main_with_errors -import pytest - - -@pytest.mark.flake8 -@pytest.mark.linter -def test_flake8(): - rc, errors = main_with_errors(argv=[]) - assert rc == 0, \ - 'Found %d code style errors / warnings:\n' % len(errors) + \ - '\n'.join(errors) diff --git a/config_utilities_launch/test/test_pep257.py b/config_utilities_launch/test/test_pep257.py deleted file mode 100644 index b234a38..0000000 --- a/config_utilities_launch/test/test_pep257.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2015 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ament_pep257.main import main -import pytest - - -@pytest.mark.linter -@pytest.mark.pep257 -def test_pep257(): - rc = main(argv=['.', 'test']) - assert rc == 0, 'Found code style errors / warnings' diff --git a/config_utilities_launch/test_launch.yaml b/config_utilities_launch/test_launch.yaml deleted file mode 100644 index 04286bc..0000000 --- a/config_utilities_launch/test_launch.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -launch: - - configured_node: - pkg: turtlesim - exec: turtlesim_node - name: sim - namespace: turtlesim1 - config_files: - - $(find-pkg-share config_utilities_launch)/some_param - - $(find-pkg-prefix config_utilities_launch)/some_other_param - config_options: - - --foo - - -bar From d640177827581e7f8924649bb97882360b879102 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Mon, 13 Jan 2025 16:41:34 +0000 Subject: [PATCH 22/28] add command line tests and fix application order --- config_utilities/src/commandline.cpp | 171 +++++++------- config_utilities/test/CMakeLists.txt | 25 +++ config_utilities/test/resources/bar.yaml | 3 + config_utilities/test/resources/foo.yaml | 4 + config_utilities/test/resources/invalid.yaml | 1 + config_utilities/test/tests/commandline.cpp | 224 +++++++++++++++++++ 6 files changed, 338 insertions(+), 90 deletions(-) create mode 100644 config_utilities/test/resources/bar.yaml create mode 100644 config_utilities/test/resources/foo.yaml create mode 100644 config_utilities/test/resources/invalid.yaml create mode 100644 config_utilities/test/tests/commandline.cpp diff --git a/config_utilities/src/commandline.cpp b/config_utilities/src/commandline.cpp index 5df4c2d..5eb335a 100644 --- a/config_utilities/src/commandline.cpp +++ b/config_utilities/src/commandline.cpp @@ -46,26 +46,32 @@ namespace internal { namespace fs = std::filesystem; +struct Span { + int pos = 0; + int num_tokens = 1; + std::string key; + + std::string extractTokens(int argc, char* argv[]) const; +}; + struct CliParser { + struct Entry { + std::string value; + bool is_file; + }; + static constexpr auto FILE_OPT = "--config-utilities-file"; static constexpr auto YAML_OPT = "--config-utilities-yaml"; CliParser() = default; CliParser& parse(int& argc, char* argv[], bool remove_args); - std::vector files; - std::vector yaml_entries; -}; - -struct Span { - int pos = 0; - int num_tokens = 1; - std::string extractTokens(int argc, char* argv[]) const; + std::vector entries; }; std::string Span::extractTokens(int argc, char* argv[]) const { if (pos < 0) { - return ""; + return ""; // NOTE(nathan) unreachable } std::stringstream ss; @@ -80,34 +86,29 @@ std::string Span::extractTokens(int argc, char* argv[]) const { return ss.str(); } -std::ostream& operator<<(std::ostream& out, const Span& span) { - out << "[" << span.pos << ", " << span.pos + span.num_tokens << "]"; - return out; -} - std::optional getSpan(int argc, char* argv[], int pos, bool parse_all, std::string& error) { int index = pos + 1; while (index < argc) { const std::string curr_opt = argv[index]; const bool is_flag = !curr_opt.empty() && curr_opt[0] == '-'; - if (is_flag && index == pos + 1) { - error = parse_all ? "at least one value required!" : "missing required value!"; - return std::nullopt; + if (is_flag) { + break; // stop parsing the span } if (!parse_all) { - return Span{pos, 1}; - } - - if (is_flag) { - break; // stop parsing the span + return Span{pos, 1, argv[pos]}; } ++index; } + if (index == pos + 1) { + error = parse_all ? "at least one value required!" : "missing required value!"; + return std::nullopt; + } + // return multi-token span - return Span{pos, index - pos - 1}; + return Span{pos, index - pos - 1, argv[pos]}; } void removeSpan(int& argc, char* argv[], const Span& span) { @@ -121,22 +122,8 @@ void removeSpan(int& argc, char* argv[], const Span& span) { argc -= span.num_tokens + 1; } -void removeSpans(int& argc, char* argv[], const std::map>& arg_spans) { - std::vector spans; - for (const auto& [key, key_spans] : arg_spans) { - for (const auto& span : key_spans) { - spans.push_back(span); - } - } - - std::sort(spans.begin(), spans.end(), [](const auto& lhs, const auto& rhs) { return lhs.pos > rhs.pos; }); - for (const auto& span : spans) { - removeSpan(argc, argv, span); - } -} - CliParser& CliParser::parse(int& argc, char* argv[], bool remove_args) { - std::map> spans; + std::vector spans; int i = 0; while (i < argc) { @@ -148,12 +135,7 @@ CliParser& CliParser::parse(int& argc, char* argv[], bool remove_args) { } if (curr_span) { - auto iter = spans.find(curr_opt); - if (iter == spans.end()) { - iter = spans.emplace(curr_opt, std::vector()).first; - } - - iter->second.emplace_back(*curr_span); + spans.emplace_back(*curr_span); i += curr_span->num_tokens; continue; } @@ -166,70 +148,79 @@ CliParser& CliParser::parse(int& argc, char* argv[], bool remove_args) { ++i; } - for (const auto& [key, key_spans] : spans) { - for (const auto& span : key_spans) { - if (key == FILE_OPT) { - files.push_back(span.extractTokens(argc, argv)); - } else { - yaml_entries.push_back(span.extractTokens(argc, argv)); - } - } + for (const auto& span : spans) { + entries.push_back(Entry{span.extractTokens(argc, argv), span.key == FILE_OPT}); } if (remove_args) { - removeSpans(argc, argv, spans); + std::sort(spans.begin(), spans.end(), [](const auto& lhs, const auto& rhs) { return lhs.pos > rhs.pos; }); + for (const auto& span : spans) { + removeSpan(argc, argv, span); + } } return *this; } -YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) { - YAML::Node node; +YAML::Node nodeFromFileEntry(const CliParser::Entry& entry) { + auto filepath = entry.value; - const auto parser = CliParser().parse(argc, argv, remove_args); + std::string ns; + const auto index = filepath.find("@"); + if (index != std::string::npos) { + ns = filepath.substr(index + 1); + filepath = filepath.substr(0, index); + } - for (const auto& entry : parser.files) { - auto filepath = entry; - std::string ns; - const auto index = filepath.find("@"); - if (index != std::string::npos) { - ns = filepath.substr(index + 1); - filepath = filepath.substr(0, index); - } + YAML::Node node; + std::filesystem::path file(filepath); + if (!fs::exists(file)) { + // TODO(nathan) log instead of manual print + std::cerr << "File " << file << " does not exist!" << std::endl; + return node; + } - std::filesystem::path file(filepath); - if (!fs::exists(file)) { - // TODO(nathan) log instead of manual print - std::cerr << "File " << file << " does not exist!" << std::endl; - continue; - } + try { + node = YAML::LoadFile(file); + } catch (const std::exception& e) { + // TODO(nathan) log instead of manual print + std::cerr << "Failure for " << file << ": " << e.what() << std::endl; + return node; + } - YAML::Node file_node; - try { - file_node = YAML::LoadFile(file); - } catch (const std::exception& e) { - // TODO(nathan) log instead of manual print - std::cerr << "Failure for " << file << ": " << e.what() << std::endl; - continue; - } + if (!ns.empty()) { + internal::moveDownNamespace(node, ns); + } - if (!ns.empty()) { - internal::moveDownNamespace(file_node, ns); - } + return node; +} - internal::mergeYamlNodes(node, file_node, true); +YAML::Node nodeFromLiteralEntry(const CliParser::Entry& entry) { + YAML::Node node; + try { + node = YAML::Load(entry.value); + } catch (const std::exception& e) { + // TODO(nathan) log instead of manual print + std::cerr << "Failure for '" << entry.value << "': " << e.what() << std::endl; } - for (const auto& entry : parser.yaml_entries) { - YAML::Node cli_node; - try { - cli_node = YAML::Load(entry); - } catch (const std::exception& e) { - // TODO(nathan) log instead of manual print - std::cerr << "Failure for '" << entry << "': " << e.what() << std::endl; + return node; +} + +YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) { + const auto parser = CliParser().parse(argc, argv, remove_args); + + YAML::Node node; + for (const auto& entry : parser.entries) { + YAML::Node parsed_node; + if (entry.is_file) { + parsed_node = nodeFromFileEntry(entry); + } else { + parsed_node = nodeFromLiteralEntry(entry); } - internal::mergeYamlNodes(node, cli_node, true); + // no-op for invalid parsed node + internal::mergeYamlNodes(node, parsed_node, true); } return node; diff --git a/config_utilities/test/CMakeLists.txt b/config_utilities/test/CMakeLists.txt index 3f638dd..05564b0 100644 --- a/config_utilities/test/CMakeLists.txt +++ b/config_utilities/test/CMakeLists.txt @@ -7,12 +7,36 @@ target_include_directories(test_${PROJECT_NAME}_plugins PRIVATE include) target_link_libraries(test_${PROJECT_NAME}_plugins ${PROJECT_NAME}) target_compile_options(test_${PROJECT_NAME}_plugins PRIVATE -fvisibility=hidden) +# Add new test resources here to ensure they get copied to the approriate build +# dir. +set(TEST_RESOURCES resources/foo.yaml resources/bar.yaml resources/invalid.yaml) + +list(TRANSFORM TEST_RESOURCES PREPEND "${CMAKE_CURRENT_BINARY_DIR}/" + OUTPUT_VARIABLE TEST_RESOURCE_BUILD_PATHS) +list(TRANSFORM TEST_RESOURCES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/" + OUTPUT_VARIABLE TEST_RESOURCE_SRC_PATHS) + +add_custom_command( + OUTPUT ${TEST_RESOURCE_BUILD_PATHS} + COMMAND ${CMAKE_COMMAND} -E make_directory + ${CMAKE_CURRENT_BINARY_DIR}/resources + COMMAND ${CMAKE_COMMAND} -E copy ${TEST_RESOURCE_SRC_PATHS} + ${CMAKE_CURRENT_BINARY_DIR}/resources/ + COMMAND_EXPAND_LISTS + COMMENT "Copy test resources to build directory" + DEPENDS ${TEST_RESOURCE_SRC_PATHS}) +add_custom_target( + copy_test_resources + COMMENT "Target for test resources" + DEPENDS ${TEST_RESOURCE_BUILD_PATHS}) + add_executable( test_${PROJECT_NAME} main.cpp src/default_config.cpp src/utils.cpp tests/asl_formatter.cpp + tests/commandline.cpp tests/config_arrays.cpp tests/config_maps.cpp tests/conversions.cpp @@ -31,6 +55,7 @@ add_executable( tests/virtual_config.cpp tests/yaml_parsing.cpp tests/yaml_utils.cpp) +add_dependencies(test_${PROJECT_NAME} copy_test_resources) target_include_directories(test_${PROJECT_NAME} PRIVATE include) target_link_libraries(test_${PROJECT_NAME} PRIVATE ${PROJECT_NAME} GTest::gtest_main) diff --git a/config_utilities/test/resources/bar.yaml b/config_utilities/test/resources/bar.yaml new file mode 100644 index 0000000..8dafb88 --- /dev/null +++ b/config_utilities/test/resources/bar.yaml @@ -0,0 +1,3 @@ +a: 6.0 +b: [4] +d: world! diff --git a/config_utilities/test/resources/foo.yaml b/config_utilities/test/resources/foo.yaml new file mode 100644 index 0000000..0f353f0 --- /dev/null +++ b/config_utilities/test/resources/foo.yaml @@ -0,0 +1,4 @@ +--- +a: 5.0 +b: [1, 2, 3] +c: hello diff --git a/config_utilities/test/resources/invalid.yaml b/config_utilities/test/resources/invalid.yaml new file mode 100644 index 0000000..d516bc0 --- /dev/null +++ b/config_utilities/test/resources/invalid.yaml @@ -0,0 +1 @@ +invalid: {incomplete: dict diff --git a/config_utilities/test/tests/commandline.cpp b/config_utilities/test/tests/commandline.cpp new file mode 100644 index 0000000..a2631e3 --- /dev/null +++ b/config_utilities/test/tests/commandline.cpp @@ -0,0 +1,224 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/parsing/commandline.h" + +#include + +#include "config_utilities/test/utils.h" + +namespace config::test { +namespace { + +struct CliArgs { + struct Args { + int argc; + char** argv; + + std::string get_cmd() const { + std::stringstream ss; + for (int i = 0; i < argc; ++i) { + ss << argv[i]; + if (i < argc - 1) { + ss << " "; + } + } + + return ss.str(); + } + }; + + explicit CliArgs(const std::vector& args) : original_args(args) { + for (auto& str : original_args) { + arg_pointers.push_back(str.data()); + } + } + + Args get() { return {static_cast(arg_pointers.size()), arg_pointers.data()}; } + + std::vector original_args; + std::vector arg_pointers; +}; + +} // namespace + +TEST(Commandline, noInputArgs) { + CliArgs cli_args(std::vector{"some_command"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const YAML::Node expected; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, invalidFlags) { + CliArgs cli_args(std::vector{"some_command", "--config-utilities-file", "--config-utilities-yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const YAML::Node expected; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command --config-utilities-file --config-utilities-yaml"); +} + +TEST(Commandline, missingFile) { + CliArgs cli_args(std::vector{"some_command", "--config-utilities-file", "resources/missing.yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const YAML::Node expected; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, invalidFile) { + CliArgs cli_args(std::vector{"some_command", "--config-utilities-file", "resources/invalid.yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const YAML::Node expected; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, vaildFiles) { + CliArgs cli_args(std::vector{"some_command", + "--config-utilities-file", + "resources/foo.yaml", + "--config-utilities-file", + "resources/bar.yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const auto expected = YAML::Load(R"yaml( +a: 6.0 +b: [1, 2, 3, 4] +c: hello +d: world! + )yaml"); + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, fileOrderingCorrect) { + CliArgs cli_args(std::vector{"some_command", + "--config-utilities-file", + "resources/bar.yaml", + "--config-utilities-file", + "resources/foo.yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, false); + const auto expected = YAML::Load(R"yaml( +a: 5.0 +b: [4, 1, 2, 3] +c: hello +d: world! + )yaml"); + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), + "some_command --config-utilities-file resources/bar.yaml --config-utilities-file resources/foo.yaml"); +} + +TEST(Commandline, mixedYamlAndFiles) { + // note that ROS can't escape correctly so args are broken by space + CliArgs cli_args(std::vector{"some_command", + "--config-utilities-file", + "resources/foo.yaml", + "--config-utilities-yaml", + "{a:", + "6.0,", + "b:", + "[4],", + "d:", + "world!}"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const auto expected = YAML::Load(R"yaml( +a: 6.0 +b: [1, 2, 3, 4] +c: hello +d: world! + )yaml"); + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, InvalidYaml) { + // note that ROS can't escape correctly so args are broken by space + CliArgs cli_args(std::vector{ + "some_command", "--config-utilities-file", "resources/foo.yaml", "--config-utilities-yaml", "{a:", "6.0,"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const auto expected = YAML::Load(R"yaml( +a: 5.0 +b: [1, 2, 3] +c: hello + )yaml"); + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, NamespacedFile) { + // note that ROS can't escape correctly so args are broken by space + // NOTE(nathan) adds intermediate flags to also check arg removal + // NOTE(nathan) order should be preserved: bar.yaml should override a for inline yaml + CliArgs cli_args(std::vector{"some_command", + "--config-utilities-file", + "resources/foo.yaml@foo/other", + "--verbose=true", + "--config-utilities-yaml", + "{c:", + "6.0, a: 7.0}", + "--some-flag", + "--config-utilities-file", + "resources/bar.yaml@", + "--ros-args", + "-r", + "other_arg:=something"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const auto expected = YAML::Load(R"yaml( +foo: + other: + a: 5.0 + b: [1, 2, 3] + c: hello +c: 6.0 +a: 6.0 +b: [4] +d: world! + )yaml"); + std::cerr << node << std::endl; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command --verbose=true --some-flag --ros-args -r other_arg:=something"); +} + +} // namespace config::test From a58abd725db3595d200e3a41a721724e97171632 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Mon, 13 Jan 2025 17:07:48 +0000 Subject: [PATCH 23/28] fix function name and add cli docs --- .../config_utilities/parsing/commandline.h | 8 ++-- docs/Parsing.md | 37 +++++++++++++++++++ docs/README.md | 1 + 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/config_utilities/include/config_utilities/parsing/commandline.h b/config_utilities/include/config_utilities/parsing/commandline.h index d922620..dca9ff2 100644 --- a/config_utilities/include/config_utilities/parsing/commandline.h +++ b/config_utilities/include/config_utilities/parsing/commandline.h @@ -116,10 +116,10 @@ std::unique_ptr createFromCLI(int argc, char* argv[], ConstructorArgument * @returns Unique pointer of type base that contains the derived object. */ template -std::unique_ptr createFromYamlWithNamespace(int argc, - char* argv[], - const std::string& name_space, - ConstructorArguments... args) { +std::unique_ptr createFromCLIWithNamespace(int argc, + char* argv[], + const std::string& name_space, + ConstructorArguments... args) { // when parsing CLI locally we don't want to modify the arguments ever const auto node = internal::loadFromArguments(argc, argv, false); const auto ns_node = internal::lookupNamespace(node, name_space); diff --git a/docs/Parsing.md b/docs/Parsing.md index 0aea783..debeec2 100644 --- a/docs/Parsing.md +++ b/docs/Parsing.md @@ -4,6 +4,7 @@ This tutorial explains how to create configs and other objects from source data. **Contents:** - [Parse from yaml](#parse-from-yaml) - [Parse from ROS](#parse-from-ros) +- [Parse from the command line](#parse-from-the-command-line) ## Parse from yaml @@ -88,3 +89,39 @@ To use [factory creation with configs](Factories.md#creating-objects-with-indivi std::unique_ptr object = createFromRosMyBase>(nh); std::unique_ptr object = createFromRosWithNamespace(nh, ns); ``` + +## Parse from the command line + +It is also possible to use the same interfaces as in the yaml or ROS case but via aggregate YAML read from the command line. To use it, include `parsing/command_line.h`. +`config_utilities` supports parsing two command line flags: + - `--config-utilities-file SOME_FILE_PATH`: Specify a file to load YAML from + - `--config-utilities-yaml SOME_ARBITRARY_YAML`: Specify YAML directly from the command line + +> **✅ Supports**
+> Note that the `--config-utilities-file` flag allows for a namespace (i.e., `some/custom/ns`) to apply to the file globally. This is specified as `--config-utilities-file SOME_FILE@some/custom/ns`. + +When aggregating the YAML from the command line, the various inputs are merged left to right (where conflicting keys from the last specified flag take precedence) and any sequences are appended together. +For those familiar with how the ROS parameter server works, this is the same behavior. +Please also note that the `--config-utilities-yaml` currently accepts multiple space-delimited tokens (because the ROS2 launch file infrastructure doesn't support escaping), so +``` +some_command --config-utilities-yaml '{my: {cool: config}}' --config-utilities-file some_file.yaml +``` +and +``` +some_command --config-utilities-yaml {my: {cool: config}} --config-utilities-file some_file.yaml +``` +will result in the same behavior (that the resulting parsed YAML will be `{my: {cool: config}}` merged with the contents of `some_file.yaml`) + +Parsing directly from the command line takes one of three forms: +```c++ +int main(int argc, char** argv) { + // Instantiate config struct directly. + MyConfig my_config = config::fromCLI(argc, argv, "optional/namespace"); + + // Factory-based instantiation (where base_args... are arguments to the object constructor) + const auto object_1 = config::createFromCLI(argc, argv, base_args...); + + // Factory-based instantiation with namespace (where base_args... are arguments to the object constructor) + const auto object_2 = config::createFromCLI(argc, argv, "optional/namespace", base_args...); +} +``` diff --git a/docs/README.md b/docs/README.md index daa6297..752db4d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,7 @@ The following tutorials will guide you through functionalities of `config_utilit 3. [**Parsing configs from data sources**](Parsing.md) - [Parse from yaml](Parsing.md#parse-from-yaml) - [Parse from ROS](Parsing.md#parse-from-ros) + - [Parse from command line](Parsing.md#parse-from-command-line) 4. [**Handling complex configs or types**](Types.md) - [Sub-configs](Types.md#sub-configs) From 1e9db514473d2fd806731f44c6dd7b12bda96e79 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Mon, 13 Jan 2025 17:11:04 +0000 Subject: [PATCH 24/28] mention multiple flags --- docs/Parsing.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/Parsing.md b/docs/Parsing.md index debeec2..518543d 100644 --- a/docs/Parsing.md +++ b/docs/Parsing.md @@ -100,9 +100,10 @@ It is also possible to use the same interfaces as in the yaml or ROS case but vi > **✅ Supports**
> Note that the `--config-utilities-file` flag allows for a namespace (i.e., `some/custom/ns`) to apply to the file globally. This is specified as `--config-utilities-file SOME_FILE@some/custom/ns`. -When aggregating the YAML from the command line, the various inputs are merged left to right (where conflicting keys from the last specified flag take precedence) and any sequences are appended together. +Both command line flags can be specified as many times as needed. +When aggregating the YAML from the command line, the various flags are merged left to right (where conflicting keys from the last specified flag take precedence) and any sequences are appended together. For those familiar with how the ROS parameter server works, this is the same behavior. -Please also note that the `--config-utilities-yaml` currently accepts multiple space-delimited tokens (because the ROS2 launch file infrastructure doesn't support escaping), so +Please also note that the `--config-utilities-yaml` currently accepts multiple space-delimited tokens (because the ROS2 launch file infrastructure does not currently correctly handle escaped substitutions), so ``` some_command --config-utilities-yaml '{my: {cool: config}}' --config-utilities-file some_file.yaml ``` From f542cb7871815ae37e30ce1aa650f330d4d70449 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Mon, 13 Jan 2025 17:44:54 +0000 Subject: [PATCH 25/28] add full api to context plus quick documentation --- config_utilities/CMakeLists.txt | 3 +- .../internal/{context.h => config_context.h} | 10 ++++ .../config_utilities/parsing/commandline.h | 8 --- .../config_utilities/parsing/context.h | 48 +++++++++++++-- config_utilities/src/commandline.cpp | 13 +--- config_utilities/src/config_context.cpp | 59 +++++++++++++++++++ config_utilities/src/context.cpp | 27 ++++----- docs/Parsing.md | 44 +++++++++++++- 8 files changed, 169 insertions(+), 43 deletions(-) rename config_utilities/include/config_utilities/internal/{context.h => config_context.h} (87%) create mode 100644 config_utilities/src/config_context.cpp diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index 8f83da3..93055b3 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -26,7 +26,8 @@ add_library( ${PROJECT_NAME} src/asl_formatter.cpp src/commandline.cpp - src/context.cpp + src/config_context.cpp # global singleton + src/context.cpp # parsing src/conversions.cpp src/external_registry.cpp src/factory.cpp diff --git a/config_utilities/include/config_utilities/internal/context.h b/config_utilities/include/config_utilities/internal/config_context.h similarity index 87% rename from config_utilities/include/config_utilities/internal/context.h rename to config_utilities/include/config_utilities/internal/config_context.h index 3c8f00c..7ef5072 100644 --- a/config_utilities/include/config_utilities/internal/context.h +++ b/config_utilities/include/config_utilities/internal/config_context.h @@ -53,11 +53,21 @@ class Context { static void update(const YAML::Node& other, const std::string& ns); + static void clear(); + + static YAML::Node toYaml(); + template static std::unique_ptr create(ConstructorArguments... args) { return internal::ObjectWithConfigFactory::create(instance().contents_, args...); } + template + static std::unique_ptr createNamespaced(const std::string& name_space, ConstructorArguments... args) { + const auto ns_node = internal::lookupNamespace(instance().contents_, name_space); + return internal::ObjectWithConfigFactory::create(ns_node, args...); + } + template static ConfigT loadConfig(const std::string& name_space = "") { ConfigT config; diff --git a/config_utilities/include/config_utilities/parsing/commandline.h b/config_utilities/include/config_utilities/parsing/commandline.h index dca9ff2..2170c4b 100644 --- a/config_utilities/include/config_utilities/parsing/commandline.h +++ b/config_utilities/include/config_utilities/parsing/commandline.h @@ -54,14 +54,6 @@ YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args); } // namespace internal -/** - * @brief Initialize global config context from the command line - * @param argc Number of arguments. - * @param argv Actual command line arguments. - * @param remove_arguments Remove parsed command line arguments. - */ -void initContext(int& argc, char* argv[], bool remove_arguments = true); - /** * @brief Loads a config based on collated YAML data specified via the command line * diff --git a/config_utilities/include/config_utilities/parsing/context.h b/config_utilities/include/config_utilities/parsing/context.h index 8a911fc..99b2c4b 100644 --- a/config_utilities/include/config_utilities/parsing/context.h +++ b/config_utilities/include/config_utilities/parsing/context.h @@ -37,10 +37,35 @@ #include -#include "config_utilities/internal/context.h" +#include "config_utilities/internal/config_context.h" namespace config { +/** + * @brief Initialize global config context from the command line + * @param argc Number of arguments. + * @param argv Actual command line arguments. + * @param remove_arguments Remove parsed command line arguments. + */ +void initContext(int& argc, char* argv[], bool remove_arguments = true); + +/** + * @brief Aggregate YAML node into global context + * @param node YAML to add to context + * @param ns Optional namespace + */ +void pushToContext(const YAML::Node& node, const std::string& ns = ""); + +/** + * @brief Delete parsed context + */ +void clearContext(); + +/** + * @brief Dump current context for exporting or saving + */ +YAML::Node contextToYaml(); + /** * @brief Loads a config from the global context * @@ -55,10 +80,7 @@ ConfigT fromContext(const std::string& name_space = "") { } /** - * @brief Create a derived type object based on a the data stored in a yaml node. All derived types need to be - * registered to the factory using a static config::Registration struct. They - * need to implement a config as a public member struct named 'Config' and use the config as the first constructor - * argument. + * @brief Create a derived type object based on currently stored YAML in config::internal::Context. * * @tparam BaseT Type of the base class to be constructed. * @tparam Args Other constructor arguments. Note that each unique set of constructor arguments will result in a @@ -71,6 +93,20 @@ std::unique_ptr createFromContext(ConstructorArguments... args) { return internal::Context::create(args...); } -// TODO(nathan) pull other options from ROS file +/** + * @brief Create a derived type object based on currently stored YAML in config::internal::Context. + * + * See createFromYamlWithNamespace() for more specific behavioral information. + * + * @tparam BaseT Type of the base class to be constructed. + * @tparam Args Other constructor arguments. + * @param name_space Optionally specify a name space to create the object from. + * @param args Other constructor arguments. + * @returns Unique pointer of type base that contains the derived object. + */ +template +std::unique_ptr createFromContextWithNamespace(const std::string& name_space, ConstructorArguments... args) { + return internal::Context::createNamespaced(name_space, args...); +} } // namespace config diff --git a/config_utilities/src/commandline.cpp b/config_utilities/src/commandline.cpp index 5eb335a..d63140d 100644 --- a/config_utilities/src/commandline.cpp +++ b/config_utilities/src/commandline.cpp @@ -38,11 +38,9 @@ #include #include -#include "config_utilities/internal/context.h" #include "config_utilities/internal/yaml_utils.h" -namespace config { -namespace internal { +namespace config::internal { namespace fs = std::filesystem; @@ -226,11 +224,4 @@ YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) { return node; } -} // namespace internal - -void initContext(int& argc, char* argv[], bool remove_arguments) { - const auto node = internal::loadFromArguments(argc, argv, remove_arguments); - internal::Context::update(node, ""); -} - -} // namespace config +} // namespace config::internal diff --git a/config_utilities/src/config_context.cpp b/config_utilities/src/config_context.cpp new file mode 100644 index 0000000..0ba4cba --- /dev/null +++ b/config_utilities/src/config_context.cpp @@ -0,0 +1,59 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/internal/config_context.h" + +#include "config_utilities/internal/yaml_utils.h" + +namespace config::internal { + +Context& Context::instance() { + static Context s_instance; + return s_instance; +} + +void Context::update(const YAML::Node& other, const std::string& ns) { + auto& context = instance(); + auto node = YAML::Clone(other); + moveDownNamespace(node, ns); + // default behavior of context is to act like the ROS1 param server and extend sequences + mergeYamlNodes(context.contents_, node, true); +} + +void Context::clear() { instance().contents_ = YAML::Node(); } + +YAML::Node Context::toYaml() { return YAML::Clone(instance().contents_); } + +} // namespace config::internal diff --git a/config_utilities/src/context.cpp b/config_utilities/src/context.cpp index f7ea373..3de9d6a 100644 --- a/config_utilities/src/context.cpp +++ b/config_utilities/src/context.cpp @@ -33,23 +33,22 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * -------------------------------------------------------------------------- */ -#include "config_utilities/internal/context.h" +#include "config_utilities/parsing/context.h" -#include "config_utilities/internal/yaml_utils.h" +#include "config_utilities/internal/config_context.h" +#include "config_utilities/parsing/commandline.h" -namespace config::internal { +namespace config { -Context& Context::instance() { - static Context s_instance; - return s_instance; +void initContext(int& argc, char* argv[], bool remove_arguments) { + const auto node = internal::loadFromArguments(argc, argv, remove_arguments); + internal::Context::update(node, ""); } -void Context::update(const YAML::Node& other, const std::string& ns) { - auto& context = instance(); - auto node = YAML::Clone(other); - moveDownNamespace(node, ns); - // default behavior of context is to act like the ROS1 param server and extend sequences - mergeYamlNodes(context.contents_, node, true); -} +void pushToContext(const YAML::Node& node, const std::string& ns) { internal::Context::update(node, ns); } + +void clearContext() { internal::Context::clear(); } + +YAML::Node contextToYaml() { return internal::Context::toYaml(); } -} // namespace config::internal +} // namespace config diff --git a/docs/Parsing.md b/docs/Parsing.md index 518543d..a20dae2 100644 --- a/docs/Parsing.md +++ b/docs/Parsing.md @@ -94,8 +94,8 @@ std::unique_ptr object = createFromRosWithNamespace(nh, ns); It is also possible to use the same interfaces as in the yaml or ROS case but via aggregate YAML read from the command line. To use it, include `parsing/command_line.h`. `config_utilities` supports parsing two command line flags: - - `--config-utilities-file SOME_FILE_PATH`: Specify a file to load YAML from - - `--config-utilities-yaml SOME_ARBITRARY_YAML`: Specify YAML directly from the command line + - `--config-utilities-file SOME_FILE_PATH`: Specify a file to load YAML from. + - `--config-utilities-yaml SOME_ARBITRARY_YAML`: Specify YAML directly from the command line. > **✅ Supports**
> Note that the `--config-utilities-file` flag allows for a namespace (i.e., `some/custom/ns`) to apply to the file globally. This is specified as `--config-utilities-file SOME_FILE@some/custom/ns`. @@ -111,7 +111,7 @@ and ``` some_command --config-utilities-yaml {my: {cool: config}} --config-utilities-file some_file.yaml ``` -will result in the same behavior (that the resulting parsed YAML will be `{my: {cool: config}}` merged with the contents of `some_file.yaml`) +will result in the same behavior (that the resulting parsed YAML will be `{my: {cool: config}}` merged with the contents of `some_file.yaml`). Parsing directly from the command line takes one of three forms: ```c++ @@ -126,3 +126,41 @@ int main(int argc, char** argv) { const auto object_2 = config::createFromCLI(argc, argv, "optional/namespace", base_args...); } ``` + +# Parsing via global context + +Usually the command line arguments or parsed YAML are not globally available to every part of an executable. +Similar to the ROS1 parameter server (and access to the parameter server by `ros::NodeHandle`), we provide a global `config::internal::Context` object (included via `parsing/context.h`) that handles tracking parsed YAML. +This `config::internal::Context` is not intended to be manipulated directly. +Instead, you should use one of the following methods: +```cpp +int main(int argc, char** argv) { + // pushes config-utilities specific flags to the end of argv and decrements argc so that it + // looks like the command was run without any config-utilities specific flags + const bool remove_config_utils_args = true; + config::initContext(argc, argv, remove_config_utils_args); + + // adds "{some: {namespace: {a: 5}}}" to the global context + config::pushToContext(YAML::Load("{a: 5}", "some/namespace")); + + // saves the loaded context + std::ofstream out("config.yaml"); + out << config::contextToYaml(); + + // clears any loaded context + config::clearContext(); +} +``` +Please note that the global context is not threadsafe. + +This object alllows instantiating configs and objects by the same three forms as the other parsing methods: +```c++ +// Instantiate config struct directly. +MyConfig my_config = config::fromContext("optional/namespace"); + +// Factory-based instantiation (where base_args... are arguments to the object constructor) +const auto object_1 = config::createFromContext(base_args...); + +// Factory-based instantiation with namespace (where base_args... are arguments to the object constructor) +const auto object_2 = config::createFromContext("optional/namespace", base_args...); +``` From 3f044fd3e3ddab6c77bb68b533317388af12ae95 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Mon, 13 Jan 2025 17:46:09 +0000 Subject: [PATCH 26/28] add doc links --- docs/Parsing.md | 3 ++- docs/README.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/Parsing.md b/docs/Parsing.md index a20dae2..1692f8a 100644 --- a/docs/Parsing.md +++ b/docs/Parsing.md @@ -5,6 +5,7 @@ This tutorial explains how to create configs and other objects from source data. - [Parse from yaml](#parse-from-yaml) - [Parse from ROS](#parse-from-ros) - [Parse from the command line](#parse-from-the-command-line) +- [Parse via global context](#parse-via-global-context) ## Parse from yaml @@ -127,7 +128,7 @@ int main(int argc, char** argv) { } ``` -# Parsing via global context +# Parse via global context Usually the command line arguments or parsed YAML are not globally available to every part of an executable. Similar to the ROS1 parameter server (and access to the parameter server by `ros::NodeHandle`), we provide a global `config::internal::Context` object (included via `parsing/context.h`) that handles tracking parsed YAML. diff --git a/docs/README.md b/docs/README.md index 752db4d..7625795 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,7 +14,8 @@ The following tutorials will guide you through functionalities of `config_utilit 3. [**Parsing configs from data sources**](Parsing.md) - [Parse from yaml](Parsing.md#parse-from-yaml) - [Parse from ROS](Parsing.md#parse-from-ros) - - [Parse from command line](Parsing.md#parse-from-command-line) + - [Parse from the command line](Parsing.md#parse-from-the-command-line) + - [Parse via global context](Parsing.md#parse-via-global-context) 4. [**Handling complex configs or types**](Types.md) - [Sub-configs](Types.md#sub-configs) From eb340982c4101049df11e07d1cdca8e409538074 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Wed, 22 Jan 2025 17:45:57 +0000 Subject: [PATCH 27/28] remove logging TODOs --- config_utilities/src/commandline.cpp | 23 ++++++++++++++--------- config_utilities/src/yaml_utils.cpp | 7 ++++++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/config_utilities/src/commandline.cpp b/config_utilities/src/commandline.cpp index d63140d..0242506 100644 --- a/config_utilities/src/commandline.cpp +++ b/config_utilities/src/commandline.cpp @@ -36,8 +36,9 @@ #include "config_utilities/parsing/commandline.h" #include -#include +#include +#include "config_utilities/internal/logger.h" #include "config_utilities/internal/yaml_utils.h" namespace config::internal { @@ -139,8 +140,9 @@ CliParser& CliParser::parse(int& argc, char* argv[], bool remove_args) { } if (!error.empty()) { - // TODO(nathan) log instead of manual print - std::cerr << "Parse issue for '" << curr_opt << "': " << error << std::endl; + std::stringstream ss; + ss << "Parse issue for '" << curr_opt << "': " << error; + Logger::logError(ss.str()); } ++i; @@ -173,16 +175,18 @@ YAML::Node nodeFromFileEntry(const CliParser::Entry& entry) { YAML::Node node; std::filesystem::path file(filepath); if (!fs::exists(file)) { - // TODO(nathan) log instead of manual print - std::cerr << "File " << file << " does not exist!" << std::endl; + std::stringstream ss; + ss << "File " << file << " does not exist!"; + Logger::logError(ss.str()); return node; } try { node = YAML::LoadFile(file); } catch (const std::exception& e) { - // TODO(nathan) log instead of manual print - std::cerr << "Failure for " << file << ": " << e.what() << std::endl; + std::stringstream ss; + ss << "Failure for " << file << ": " << e.what(); + Logger::logError(ss.str()); return node; } @@ -198,8 +202,9 @@ YAML::Node nodeFromLiteralEntry(const CliParser::Entry& entry) { try { node = YAML::Load(entry.value); } catch (const std::exception& e) { - // TODO(nathan) log instead of manual print - std::cerr << "Failure for '" << entry.value << "': " << e.what() << std::endl; + std::stringstream ss; + ss << "Failure for '" << entry.value << "': " << e.what(); + Logger::logError(ss.str()); } return node; diff --git a/config_utilities/src/yaml_utils.cpp b/config_utilities/src/yaml_utils.cpp index 1e97e18..e443ff0 100644 --- a/config_utilities/src/yaml_utils.cpp +++ b/config_utilities/src/yaml_utils.cpp @@ -35,6 +35,9 @@ #include "config_utilities/internal/yaml_utils.h" +#include + +#include "config_utilities/internal/logger.h" #include "config_utilities/internal/string_utils.h" namespace config::internal { @@ -70,7 +73,9 @@ void mergeYamlNodes(YAML::Node& a, const YAML::Node& b, bool extend_sequences) { // Both a and b are maps: merge all entries of b into a. for (const auto& node : b) { if (!node.first.IsScalar()) { - // TODO(nathan) warn about not handling complex keys + std::stringstream ss; + ss << "Complex keys not supported, dropping '" << node.first << "' during merge"; + Logger::logWarning(ss.str()); continue; } From e49a47dfb6d9a8c4d5438ee3addbd97883b39fd9 Mon Sep 17 00:00:00 2001 From: Nathan Hughes Date: Wed, 22 Jan 2025 18:50:55 +0000 Subject: [PATCH 28/28] log to stderr for errors and warnings by default --- .../config_utilities/internal/logger.h | 4 ++-- .../config_utilities/logging/log_to_stdout.h | 10 +++++++- .../include/config_utilities/settings.h | 3 +++ config_utilities/src/log_to_stdout.cpp | 24 ++++++++++++++----- config_utilities/src/logger.cpp | 6 ++++- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/config_utilities/include/config_utilities/internal/logger.h b/config_utilities/include/config_utilities/internal/logger.h index b337120..6504136 100644 --- a/config_utilities/include/config_utilities/internal/logger.h +++ b/config_utilities/include/config_utilities/internal/logger.h @@ -42,8 +42,8 @@ namespace config::internal { -// Enum for different severity levels of logging. -enum class Severity { kInfo, kWarning, kError, kFatal }; +// Enum for different severity levels of logging. Enum values are used for comparing logging levels. +enum class Severity { kInfo = 0, kWarning = 1, kError = 2, kFatal = 3 }; std::string severityToString(const Severity severity); diff --git a/config_utilities/include/config_utilities/logging/log_to_stdout.h b/config_utilities/include/config_utilities/logging/log_to_stdout.h index dd09fad..67c574b 100644 --- a/config_utilities/include/config_utilities/logging/log_to_stdout.h +++ b/config_utilities/include/config_utilities/logging/log_to_stdout.h @@ -45,13 +45,21 @@ namespace config::internal { */ class StdoutLogger : public Logger { public: - StdoutLogger() = default; + /** + * @brief Construct a logger to output to stdout or stderr depending on configuration + * @param min_severity Mininum severity to output + * @param stderr_severity Mininum severity to log to stderr instead of stdout + */ + StdoutLogger(Severity min_severity = Severity::kWarning, Severity stderr_severity = Severity::kError); + virtual ~StdoutLogger() = default; protected: void logImpl(const Severity severity, const std::string& message) override; private: + const Severity min_severity_; + const Severity stderr_severity_; // Factory registration to allow setting of formatters via Settings::setLogger(). inline static const auto registration_ = Registration("stdout"); diff --git a/config_utilities/include/config_utilities/settings.h b/config_utilities/include/config_utilities/settings.h index 4371843..78ec304 100644 --- a/config_utilities/include/config_utilities/settings.h +++ b/config_utilities/include/config_utilities/settings.h @@ -91,6 +91,9 @@ struct Settings { //! @brief Log any factory creation from an external library (for debugging purposes) bool print_external_allocations = false; + //! @brief Control whether config_utilities is initialized to log to stdout/stderr by default + bool disable_default_stdout_logger = false; + /* Options to specify the logger and formatter at run time. */ // Specify the default logger to be used for printing. Loggers register themselves if included. void setLogger(const std::string& name); diff --git a/config_utilities/src/log_to_stdout.cpp b/config_utilities/src/log_to_stdout.cpp index 274365b..64466d1 100644 --- a/config_utilities/src/log_to_stdout.cpp +++ b/config_utilities/src/log_to_stdout.cpp @@ -40,27 +40,39 @@ namespace config::internal { +StdoutLogger::StdoutLogger(Severity min_severity, Severity stderr_severity) + : min_severity_(min_severity), stderr_severity_(stderr_severity) {} + void StdoutLogger::logImpl(const Severity severity, const std::string& message) { + if (severity < min_severity_ && severity != Severity::kFatal) { + return; + } + + std::stringstream ss; switch (severity) { case Severity::kInfo: - std::cout << "[INFO] " << message << std::endl; + ss << "[INFO] " << message; break; case Severity::kWarning: - std::cout << "\033[33m[WARNING] " << message << "\033[0m" << std::endl; + ss << "\033[33m[WARNING] " << message << "\033[0m"; break; case Severity::kError: - std::cout << "\033[31m[ERROR] " << message << "\033[0m" << std::endl; + ss << "\033[31m[ERROR] " << message << "\033[0m"; break; case Severity::kFatal: throw std::runtime_error(message); } -} -StdoutLogger::Initializer::Initializer() { - Logger::setLogger(std::make_shared()); + if (severity < stderr_severity_) { + std::cout << ss.str() << std::endl; + } else { + std::cerr << ss.str() << std::endl; + } } +StdoutLogger::Initializer::Initializer() { Logger::setLogger(std::make_shared()); } + } // namespace config::internal diff --git a/config_utilities/src/logger.cpp b/config_utilities/src/logger.cpp index 4a002ea..f7bc72a 100644 --- a/config_utilities/src/logger.cpp +++ b/config_utilities/src/logger.cpp @@ -37,6 +37,8 @@ #include +#include "config_utilities/logging/log_to_stdout.h" + namespace config::internal { Logger::Ptr Logger::logger_; @@ -73,7 +75,9 @@ void Logger::setLogger(Logger::Ptr logger) { void Logger::dispatch(const Severity severity, const std::string& message) { if (!logger_) { - logger_ = std::make_shared(); + // NOTE(nathan) we default to logging to stdout/stderr to make sure warnings and errors are visible + logger_ = Settings::instance().disable_default_stdout_logger ? std::make_shared() + : std::make_shared(); } logger_->logImpl(severity, message);