diff --git a/config_utilities/CMakeLists.txt b/config_utilities/CMakeLists.txt index 8b5faa8..93055b3 100644 --- a/config_utilities/CMakeLists.txt +++ b/config_utilities/CMakeLists.txt @@ -14,7 +14,6 @@ option(CONFIG_UTILS_ENABLE_ROS "Export roscpp and build related code" ON) option(CONFIG_UTILS_ENABLE_EIGEN "Export Eigen and build related code" ON) option(CONFIG_UTILS_ENABLE_GLOG "Export glog and build related code" ON) option(CONFIG_UTILS_BUILD_TESTS "Build unit tests" ON) -option(CONFIG_UTILS_INSTALL_TESTS "Install test executable" ON) option(CONFIG_UTILS_BUILD_DEMOS "Build demo executables" ON) find_package(yaml-cpp REQUIRED) @@ -26,6 +25,9 @@ find_optional_pkgcfg(libglog CONFIG_UTILS_ENABLE_GLOG) add_library( ${PROJECT_NAME} src/asl_formatter.cpp + src/commandline.cpp + src/config_context.cpp # global singleton + src/context.cpp # parsing src/conversions.cpp src/external_registry.cpp src/factory.cpp @@ -74,6 +76,7 @@ if(CONFIG_UTILS_BUILD_DEMOS) endif() if(CONFIG_UTILS_BUILD_TESTS) + include(CTest) enable_testing() add_subdirectory(test) endif() diff --git a/config_utilities/cmake/OptionalPackage.cmake b/config_utilities/cmake/OptionalPackage.cmake index 83fd50b..33eb365 100644 --- a/config_utilities/cmake/OptionalPackage.cmake +++ b/config_utilities/cmake/OptionalPackage.cmake @@ -8,7 +8,7 @@ # not the package should be used macro(FIND_OPTIONAL package_name package_enabled) if(${package_enabled}) - find_package(${package_name}) + find_package(${package_name} QUIET) endif() if(${${package_name}_FOUND}) diff --git a/config_utilities/cmake/config_utilitiesConfig.cmake.in b/config_utilities/cmake/config_utilitiesConfig.cmake.in index 756a3cf..0d0e3f6 100644 --- a/config_utilities/cmake/config_utilitiesConfig.cmake.in +++ b/config_utilities/cmake/config_utilitiesConfig.cmake.in @@ -1,6 +1,7 @@ @PACKAGE_INIT@ -get_filename_component(config_utilities_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) +get_filename_component(config_utilities_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" + PATH) include(CMakeFindDependencyMacro) find_dependency(yaml-cpp REQUIRED) @@ -11,6 +12,11 @@ endif() if(@ENABLE_roscpp@) find_dependency(roscpp REQUIRED) + set(config_utilities_FOUND_CATKIN_PROJECT TRUE) +endif() + +if(@ENABLE_rclcpp@) + find_dependency(rclcpp REQUIRED) endif() if(@ENABLE_libglog@) @@ -24,6 +30,4 @@ endif() set(config_utilities_LIBRARIES config_utilities::config_utilities) -set(config_utilities_FOUND_CATKIN_PROJECT TRUE) - check_required_components(config_utilities) diff --git a/config_utilities/include/config_utilities/internal/checks.h b/config_utilities/include/config_utilities/internal/checks.h index 846aad0..e234e4e 100644 --- a/config_utilities/include/config_utilities/internal/checks.h +++ b/config_utilities/include/config_utilities/internal/checks.h @@ -35,6 +35,7 @@ #pragma once +#include #include #include #include @@ -82,7 +83,7 @@ struct CompareMessageTrait { template class BinaryCheck : public CheckBase { public: - BinaryCheck(const T& param, const T& value, const std::string name = "") + BinaryCheck(const T& param, const T& value, const std::string& name = "") : param_(param), value_(value), name_(name) {} bool valid() const override { return Compare{}(param_, value_); } @@ -186,12 +187,8 @@ class CheckIsOneOf : public CheckBase { : param_(param), candidates_(candidates), name_(name) {} bool valid() const override { - for (const T& cadidate : candidates_) { - if (param_ == cadidate) { - return true; - } - } - return false; + // check that param matches any candidate + return std::any_of(candidates_.begin(), candidates_.end(), [this](const auto& c) { return c == param_; }); } std::string message() const override { diff --git a/config_utilities/include/config_utilities/internal/config_context.h b/config_utilities/include/config_utilities/internal/config_context.h new file mode 100644 index 0000000..7ef5072 --- /dev/null +++ b/config_utilities/include/config_utilities/internal/config_context.h @@ -0,0 +1,85 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#pragma once + +#include + +#include + +#include "config_utilities/factory.h" +#include "config_utilities/internal/visitor.h" + +namespace config::internal { + +/** + * @brief Context is a singleton that holds the raw parsed information used to generate configs + */ +class Context { + public: + ~Context() = default; + + static void update(const YAML::Node& other, const std::string& ns); + + static void clear(); + + static YAML::Node toYaml(); + + template + static std::unique_ptr create(ConstructorArguments... args) { + return internal::ObjectWithConfigFactory::create(instance().contents_, args...); + } + + template + static std::unique_ptr createNamespaced(const std::string& name_space, ConstructorArguments... args) { + const auto ns_node = internal::lookupNamespace(instance().contents_, name_space); + return internal::ObjectWithConfigFactory::create(ns_node, args...); + } + + template + static ConfigT loadConfig(const std::string& name_space = "") { + ConfigT config; + internal::Visitor::setValues(config, internal::lookupNamespace(instance().contents_, name_space), true); + return config; + } + + private: + Context() = default; + static Context& instance(); + + YAML::Node contents_; +}; + +} // namespace config::internal diff --git a/config_utilities/include/config_utilities/internal/logger.h b/config_utilities/include/config_utilities/internal/logger.h index b337120..6504136 100644 --- a/config_utilities/include/config_utilities/internal/logger.h +++ b/config_utilities/include/config_utilities/internal/logger.h @@ -42,8 +42,8 @@ namespace config::internal { -// Enum for different severity levels of logging. -enum class Severity { kInfo, kWarning, kError, kFatal }; +// Enum for different severity levels of logging. Enum values are used for comparing logging levels. +enum class Severity { kInfo = 0, kWarning = 1, kError = 2, kFatal = 3 }; std::string severityToString(const Severity severity); diff --git a/config_utilities/include/config_utilities/internal/yaml_utils.h b/config_utilities/include/config_utilities/internal/yaml_utils.h index 2d0fe70..2e35db9 100644 --- a/config_utilities/include/config_utilities/internal/yaml_utils.h +++ b/config_utilities/include/config_utilities/internal/yaml_utils.h @@ -44,9 +44,10 @@ namespace config::internal { /** * @brief Merges node b into a, overwriting values previously defined in a if they can not be - * merged. Modifies node a, whereas b is const. + * merged. Modifies node a, whereas b is const. Sequences can optionally be appended together at the same level of the + * YAML tree. */ -void mergeYamlNodes(YAML::Node& a, const YAML::Node& b); +void mergeYamlNodes(YAML::Node& a, const YAML::Node& b, bool extend_sequences = false); /** * @brief Get a pointer to the final node of the specified namespace if it exists, where each map in the yaml is diff --git a/config_utilities/include/config_utilities/logging/log_to_stdout.h b/config_utilities/include/config_utilities/logging/log_to_stdout.h index dd09fad..67c574b 100644 --- a/config_utilities/include/config_utilities/logging/log_to_stdout.h +++ b/config_utilities/include/config_utilities/logging/log_to_stdout.h @@ -45,13 +45,21 @@ namespace config::internal { */ class StdoutLogger : public Logger { public: - StdoutLogger() = default; + /** + * @brief Construct a logger to output to stdout or stderr depending on configuration + * @param min_severity Mininum severity to output + * @param stderr_severity Mininum severity to log to stderr instead of stdout + */ + StdoutLogger(Severity min_severity = Severity::kWarning, Severity stderr_severity = Severity::kError); + virtual ~StdoutLogger() = default; protected: void logImpl(const Severity severity, const std::string& message) override; private: + const Severity min_severity_; + const Severity stderr_severity_; // Factory registration to allow setting of formatters via Settings::setLogger(). inline static const auto registration_ = Registration("stdout"); diff --git a/config_utilities/include/config_utilities/parsing/commandline.h b/config_utilities/include/config_utilities/parsing/commandline.h new file mode 100644 index 0000000..2170c4b --- /dev/null +++ b/config_utilities/include/config_utilities/parsing/commandline.h @@ -0,0 +1,121 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#pragma once + +#include +#include + +#include "config_utilities/factory.h" +#include "config_utilities/internal/visitor.h" +#include "config_utilities/internal/yaml_utils.h" + +namespace config { +namespace internal { + +/** + * @brief Parse and collate YAML node from arguments, optionally removing arguments + * @param argc Number of command line arguments + * @param argv Command line argument strings + */ +YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args); + +} // namespace internal + +/** + * @brief Loads a config based on collated YAML data specified via the command line + * + * See fromYaml() for more specific behavioral information. + * + * @tparam ConfigT The config type. This can also be a VirtualConfig or a std::vector. + * @param argc Number of arguments. + * @param argv Actual command line arguments. + * @param name_space Optional namespace to use under the resolved YAML parameter tree. + * @returns The config. + */ +template +ConfigT fromCLI(int argc, char* argv[], const std::string& name_space = "") { + // when parsing CLI locally we don't want to modify the arguments ever + const auto node = internal::loadFromArguments(argc, argv, false); + + ConfigT config; + internal::Visitor::setValues(config, internal::lookupNamespace(node, name_space), true); + return config; +} + +/** + * @brief Create a derived type object based on collated YAML data specified via the command line + * + * See createFromYaml() for more specific behavioral information. + * + * @tparam BaseT Type of the base class to be constructed. + * @tparam Args Other constructor arguments. + * @param argc Number of arguments. + * @param argv Actual command line arguments. + * @param args Other constructor arguments. + * @returns Unique pointer of type base that contains the derived object. + */ +template +std::unique_ptr createFromCLI(int argc, char* argv[], ConstructorArguments... args) { + // when parsing CLI locally we don't want to modify the arguments ever + const auto node = internal::loadFromArguments(argc, argv, false); + return internal::ObjectWithConfigFactory::create(node, args...); +} + +/** + * @brief Create a derived type object based on collated YAML data specified via the command line + * + * See createFromYamlWithNamespace() for more specific behavioral information. + * + * @tparam BaseT Type of the base class to be constructed. + * @tparam Args Other constructor arguments. + * @param argc Number of arguments. + * @param argv Actual command line arguments. + * @param name_space Optionally specify a name space to create the object from. + * @param args Other constructor arguments. + * @returns Unique pointer of type base that contains the derived object. + */ +template +std::unique_ptr createFromCLIWithNamespace(int argc, + char* argv[], + const std::string& name_space, + ConstructorArguments... args) { + // when parsing CLI locally we don't want to modify the arguments ever + const auto node = internal::loadFromArguments(argc, argv, false); + const auto ns_node = internal::lookupNamespace(node, name_space); + return internal::ObjectWithConfigFactory::create(ns_node, args...); +} + +} // namespace config diff --git a/config_utilities/include/config_utilities/parsing/context.h b/config_utilities/include/config_utilities/parsing/context.h new file mode 100644 index 0000000..99b2c4b --- /dev/null +++ b/config_utilities/include/config_utilities/parsing/context.h @@ -0,0 +1,112 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#pragma once + +#include + +#include "config_utilities/internal/config_context.h" + +namespace config { + +/** + * @brief Initialize global config context from the command line + * @param argc Number of arguments. + * @param argv Actual command line arguments. + * @param remove_arguments Remove parsed command line arguments. + */ +void initContext(int& argc, char* argv[], bool remove_arguments = true); + +/** + * @brief Aggregate YAML node into global context + * @param node YAML to add to context + * @param ns Optional namespace + */ +void pushToContext(const YAML::Node& node, const std::string& ns = ""); + +/** + * @brief Delete parsed context + */ +void clearContext(); + +/** + * @brief Dump current context for exporting or saving + */ +YAML::Node contextToYaml(); + +/** + * @brief Loads a config from the global context + * + * @tparam ConfigT The config type. This can also be a VirtualConfig or a std::vector. + * @param name_space Optionally specify a name space to create the config from. Separate names with slashes '/'. + * Example: "my_config/my_sub_config". + * @returns The config. + */ +template +ConfigT fromContext(const std::string& name_space = "") { + return internal::Context::loadConfig(name_space); +} + +/** + * @brief Create a derived type object based on currently stored YAML in config::internal::Context. + * + * @tparam BaseT Type of the base class to be constructed. + * @tparam Args Other constructor arguments. Note that each unique set of constructor arguments will result in a + * different base-entry in the factory. + * @param args Other constructor arguments. + * @returns Unique pointer of type base that contains the derived object. + */ +template +std::unique_ptr createFromContext(ConstructorArguments... args) { + return internal::Context::create(args...); +} + +/** + * @brief Create a derived type object based on currently stored YAML in config::internal::Context. + * + * See createFromYamlWithNamespace() for more specific behavioral information. + * + * @tparam BaseT Type of the base class to be constructed. + * @tparam Args Other constructor arguments. + * @param name_space Optionally specify a name space to create the object from. + * @param args Other constructor arguments. + * @returns Unique pointer of type base that contains the derived object. + */ +template +std::unique_ptr createFromContextWithNamespace(const std::string& name_space, ConstructorArguments... args) { + return internal::Context::createNamespaced(name_space, args...); +} + +} // namespace config diff --git a/config_utilities/include/config_utilities/settings.h b/config_utilities/include/config_utilities/settings.h index 4371843..78ec304 100644 --- a/config_utilities/include/config_utilities/settings.h +++ b/config_utilities/include/config_utilities/settings.h @@ -91,6 +91,9 @@ struct Settings { //! @brief Log any factory creation from an external library (for debugging purposes) bool print_external_allocations = false; + //! @brief Control whether config_utilities is initialized to log to stdout/stderr by default + bool disable_default_stdout_logger = false; + /* Options to specify the logger and formatter at run time. */ // Specify the default logger to be used for printing. Loggers register themselves if included. void setLogger(const std::string& name); diff --git a/config_utilities/package.xml b/config_utilities/package.xml index 58039b8..26ee33f 100644 --- a/config_utilities/package.xml +++ b/config_utilities/package.xml @@ -13,6 +13,7 @@ BSD-3-Clause cmake + yaml-cpp cmake diff --git a/config_utilities/src/commandline.cpp b/config_utilities/src/commandline.cpp new file mode 100644 index 0000000..0242506 --- /dev/null +++ b/config_utilities/src/commandline.cpp @@ -0,0 +1,232 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/parsing/commandline.h" + +#include +#include + +#include "config_utilities/internal/logger.h" +#include "config_utilities/internal/yaml_utils.h" + +namespace config::internal { + +namespace fs = std::filesystem; + +struct Span { + int pos = 0; + int num_tokens = 1; + std::string key; + + std::string extractTokens(int argc, char* argv[]) const; +}; + +struct CliParser { + struct Entry { + std::string value; + bool is_file; + }; + + static constexpr auto FILE_OPT = "--config-utilities-file"; + static constexpr auto YAML_OPT = "--config-utilities-yaml"; + + CliParser() = default; + CliParser& parse(int& argc, char* argv[], bool remove_args); + + std::vector entries; +}; + +std::string Span::extractTokens(int argc, char* argv[]) const { + if (pos < 0) { + return ""; // NOTE(nathan) unreachable + } + + std::stringstream ss; + const auto total = std::min(argc, pos + num_tokens + 1); + for (int i = pos + 1; i < total; ++i) { + ss << argv[i]; + if (i < total - 1) { + ss << " "; + } + } + + return ss.str(); +} + +std::optional getSpan(int argc, char* argv[], int pos, bool parse_all, std::string& error) { + int index = pos + 1; + while (index < argc) { + const std::string curr_opt = argv[index]; + const bool is_flag = !curr_opt.empty() && curr_opt[0] == '-'; + if (is_flag) { + break; // stop parsing the span + } + + if (!parse_all) { + return Span{pos, 1, argv[pos]}; + } + + ++index; + } + + if (index == pos + 1) { + error = parse_all ? "at least one value required!" : "missing required value!"; + return std::nullopt; + } + + // return multi-token span + return Span{pos, index - pos - 1, argv[pos]}; +} + +void removeSpan(int& argc, char* argv[], const Span& span) { + for (int token = span.num_tokens; token >= 0; --token) { + for (int i = span.pos + token; i < argc - (span.num_tokens - token); ++i) { + // bubble-sort esque shuffle to move args to end + std::swap(argv[i], argv[i + 1]); + } + } + + argc -= span.num_tokens + 1; +} + +CliParser& CliParser::parse(int& argc, char* argv[], bool remove_args) { + std::vector spans; + + int i = 0; + while (i < argc) { + const std::string curr_opt(argv[i]); + std::string error; + std::optional curr_span; + if (curr_opt == FILE_OPT || curr_opt == YAML_OPT) { + curr_span = getSpan(argc, argv, i, curr_opt == YAML_OPT, error); + } + + if (curr_span) { + spans.emplace_back(*curr_span); + i += curr_span->num_tokens; + continue; + } + + if (!error.empty()) { + std::stringstream ss; + ss << "Parse issue for '" << curr_opt << "': " << error; + Logger::logError(ss.str()); + } + + ++i; + } + + for (const auto& span : spans) { + entries.push_back(Entry{span.extractTokens(argc, argv), span.key == FILE_OPT}); + } + + if (remove_args) { + std::sort(spans.begin(), spans.end(), [](const auto& lhs, const auto& rhs) { return lhs.pos > rhs.pos; }); + for (const auto& span : spans) { + removeSpan(argc, argv, span); + } + } + + return *this; +} + +YAML::Node nodeFromFileEntry(const CliParser::Entry& entry) { + auto filepath = entry.value; + + std::string ns; + const auto index = filepath.find("@"); + if (index != std::string::npos) { + ns = filepath.substr(index + 1); + filepath = filepath.substr(0, index); + } + + YAML::Node node; + std::filesystem::path file(filepath); + if (!fs::exists(file)) { + std::stringstream ss; + ss << "File " << file << " does not exist!"; + Logger::logError(ss.str()); + return node; + } + + try { + node = YAML::LoadFile(file); + } catch (const std::exception& e) { + std::stringstream ss; + ss << "Failure for " << file << ": " << e.what(); + Logger::logError(ss.str()); + return node; + } + + if (!ns.empty()) { + internal::moveDownNamespace(node, ns); + } + + return node; +} + +YAML::Node nodeFromLiteralEntry(const CliParser::Entry& entry) { + YAML::Node node; + try { + node = YAML::Load(entry.value); + } catch (const std::exception& e) { + std::stringstream ss; + ss << "Failure for '" << entry.value << "': " << e.what(); + Logger::logError(ss.str()); + } + + return node; +} + +YAML::Node loadFromArguments(int& argc, char* argv[], bool remove_args) { + const auto parser = CliParser().parse(argc, argv, remove_args); + + YAML::Node node; + for (const auto& entry : parser.entries) { + YAML::Node parsed_node; + if (entry.is_file) { + parsed_node = nodeFromFileEntry(entry); + } else { + parsed_node = nodeFromLiteralEntry(entry); + } + + // no-op for invalid parsed node + internal::mergeYamlNodes(node, parsed_node, true); + } + + return node; +} + +} // namespace config::internal diff --git a/config_utilities/src/config_context.cpp b/config_utilities/src/config_context.cpp new file mode 100644 index 0000000..0ba4cba --- /dev/null +++ b/config_utilities/src/config_context.cpp @@ -0,0 +1,59 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/internal/config_context.h" + +#include "config_utilities/internal/yaml_utils.h" + +namespace config::internal { + +Context& Context::instance() { + static Context s_instance; + return s_instance; +} + +void Context::update(const YAML::Node& other, const std::string& ns) { + auto& context = instance(); + auto node = YAML::Clone(other); + moveDownNamespace(node, ns); + // default behavior of context is to act like the ROS1 param server and extend sequences + mergeYamlNodes(context.contents_, node, true); +} + +void Context::clear() { instance().contents_ = YAML::Node(); } + +YAML::Node Context::toYaml() { return YAML::Clone(instance().contents_); } + +} // namespace config::internal diff --git a/config_utilities/src/context.cpp b/config_utilities/src/context.cpp new file mode 100644 index 0000000..3de9d6a --- /dev/null +++ b/config_utilities/src/context.cpp @@ -0,0 +1,54 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/parsing/context.h" + +#include "config_utilities/internal/config_context.h" +#include "config_utilities/parsing/commandline.h" + +namespace config { + +void initContext(int& argc, char* argv[], bool remove_arguments) { + const auto node = internal::loadFromArguments(argc, argv, remove_arguments); + internal::Context::update(node, ""); +} + +void pushToContext(const YAML::Node& node, const std::string& ns) { internal::Context::update(node, ns); } + +void clearContext() { internal::Context::clear(); } + +YAML::Node contextToYaml() { return internal::Context::toYaml(); } + +} // namespace config diff --git a/config_utilities/src/external_registry.cpp b/config_utilities/src/external_registry.cpp index 583bce2..a686dbd 100644 --- a/config_utilities/src/external_registry.cpp +++ b/config_utilities/src/external_registry.cpp @@ -98,10 +98,12 @@ LibraryGuard::operator bool() const { return !libraries_.empty(); } ExternalRegistry::~ExternalRegistry() { std::vector libraries; for (const auto& [path, lib] : libraries_) { + // NOTE(nathan) technically unreachable (library guards call unload first) libraries.push_back(path); } for (const auto& path : libraries) { + // NOTE(nathan) technically unreachable (library guards call unload first) unload(path); } } diff --git a/config_utilities/src/log_to_stdout.cpp b/config_utilities/src/log_to_stdout.cpp index 274365b..64466d1 100644 --- a/config_utilities/src/log_to_stdout.cpp +++ b/config_utilities/src/log_to_stdout.cpp @@ -40,27 +40,39 @@ namespace config::internal { +StdoutLogger::StdoutLogger(Severity min_severity, Severity stderr_severity) + : min_severity_(min_severity), stderr_severity_(stderr_severity) {} + void StdoutLogger::logImpl(const Severity severity, const std::string& message) { + if (severity < min_severity_ && severity != Severity::kFatal) { + return; + } + + std::stringstream ss; switch (severity) { case Severity::kInfo: - std::cout << "[INFO] " << message << std::endl; + ss << "[INFO] " << message; break; case Severity::kWarning: - std::cout << "\033[33m[WARNING] " << message << "\033[0m" << std::endl; + ss << "\033[33m[WARNING] " << message << "\033[0m"; break; case Severity::kError: - std::cout << "\033[31m[ERROR] " << message << "\033[0m" << std::endl; + ss << "\033[31m[ERROR] " << message << "\033[0m"; break; case Severity::kFatal: throw std::runtime_error(message); } -} -StdoutLogger::Initializer::Initializer() { - Logger::setLogger(std::make_shared()); + if (severity < stderr_severity_) { + std::cout << ss.str() << std::endl; + } else { + std::cerr << ss.str() << std::endl; + } } +StdoutLogger::Initializer::Initializer() { Logger::setLogger(std::make_shared()); } + } // namespace config::internal diff --git a/config_utilities/src/logger.cpp b/config_utilities/src/logger.cpp index 4a002ea..f7bc72a 100644 --- a/config_utilities/src/logger.cpp +++ b/config_utilities/src/logger.cpp @@ -37,6 +37,8 @@ #include +#include "config_utilities/logging/log_to_stdout.h" + namespace config::internal { Logger::Ptr Logger::logger_; @@ -73,7 +75,9 @@ void Logger::setLogger(Logger::Ptr logger) { void Logger::dispatch(const Severity severity, const std::string& message) { if (!logger_) { - logger_ = std::make_shared(); + // NOTE(nathan) we default to logging to stdout/stderr to make sure warnings and errors are visible + logger_ = Settings::instance().disable_default_stdout_logger ? std::make_shared() + : std::make_shared(); } logger_->logImpl(severity, message); diff --git a/config_utilities/src/yaml_utils.cpp b/config_utilities/src/yaml_utils.cpp index 6eb5ae9..e443ff0 100644 --- a/config_utilities/src/yaml_utils.cpp +++ b/config_utilities/src/yaml_utils.cpp @@ -35,40 +35,58 @@ #include "config_utilities/internal/yaml_utils.h" +#include + +#include "config_utilities/internal/logger.h" #include "config_utilities/internal/string_utils.h" namespace config::internal { +namespace { -void mergeYamlNodes(YAML::Node& a, const YAML::Node& b) { - if (!b.IsMap()) { - // If b is not a map, merge result is b, unless b is null. - if (b.IsNull() || !b.IsDefined()) { - return; - } - a = YAML::Clone(b); +inline void mergeLeaves(YAML::Node& a, const YAML::Node& b, bool extend_sequences) { + // If b is invalid, we can't do anything. + if (b.IsNull() || !b.IsDefined()) { return; } - if (!a.IsMap()) { - // If a is not a map, merge result is b + + // If either a or b is not a sequence, assign b to a (or if extending is disabled) + if (!extend_sequences || !a.IsSequence() || !b.IsSequence()) { a = YAML::Clone(b); return; } - if (!b.size()) { - // If a is a map, and b is an empty map, return a + + // both a and b are sequences: append b to a. + for (const auto& child : b) { + a.push_back(YAML::Clone(child)); + } +} + +} // namespace + +void mergeYamlNodes(YAML::Node& a, const YAML::Node& b, bool extend_sequences) { + // If either node is a leaf in the config tree, pass merging behavior to helper function + if (!b.IsMap() || !a.IsMap()) { + mergeLeaves(a, b, extend_sequences); return; } - // Merge all entries of b into a. - for (const auto kv_pair : b) { - if (kv_pair.first.IsScalar()) { - const std::string& key = kv_pair.first.Scalar(); - if (a[key]) { - // Node exists. Merge recursively. - YAML::Node a_sub = a[key]; // This node is a ref. - mergeYamlNodes(a_sub, kv_pair.second); - } else { - a[key] = YAML::Clone(kv_pair.second); - } + // Both a and b are maps: merge all entries of b into a. + for (const auto& node : b) { + if (!node.first.IsScalar()) { + std::stringstream ss; + ss << "Complex keys not supported, dropping '" << node.first << "' during merge"; + Logger::logWarning(ss.str()); + continue; + } + + const auto& key = node.first.Scalar(); + if (a[key]) { + // Node exists. Merge recursively. + YAML::Node a_sub = a[key]; // This node is a ref. + mergeYamlNodes(a_sub, node.second, extend_sequences); + } else { + // Leaf of a, but b continues: insert b + a[key] = YAML::Clone(node.second); } } } @@ -98,6 +116,7 @@ bool isEqual(const YAML::Node& a, const YAML::Node& b) { if (a.Type() != b.Type()) { return false; } + switch (a.Type()) { case YAML::NodeType::Scalar: return a.Scalar() == b.Scalar(); @@ -126,10 +145,10 @@ bool isEqual(const YAML::Node& a, const YAML::Node& b) { } return true; case YAML::NodeType::Null: - return true; case YAML::NodeType::Undefined: return true; } + return false; } diff --git a/config_utilities/test/CMakeLists.txt b/config_utilities/test/CMakeLists.txt index 2f0061e..05564b0 100644 --- a/config_utilities/test/CMakeLists.txt +++ b/config_utilities/test/CMakeLists.txt @@ -7,12 +7,36 @@ target_include_directories(test_${PROJECT_NAME}_plugins PRIVATE include) target_link_libraries(test_${PROJECT_NAME}_plugins ${PROJECT_NAME}) target_compile_options(test_${PROJECT_NAME}_plugins PRIVATE -fvisibility=hidden) +# Add new test resources here to ensure they get copied to the approriate build +# dir. +set(TEST_RESOURCES resources/foo.yaml resources/bar.yaml resources/invalid.yaml) + +list(TRANSFORM TEST_RESOURCES PREPEND "${CMAKE_CURRENT_BINARY_DIR}/" + OUTPUT_VARIABLE TEST_RESOURCE_BUILD_PATHS) +list(TRANSFORM TEST_RESOURCES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/" + OUTPUT_VARIABLE TEST_RESOURCE_SRC_PATHS) + +add_custom_command( + OUTPUT ${TEST_RESOURCE_BUILD_PATHS} + COMMAND ${CMAKE_COMMAND} -E make_directory + ${CMAKE_CURRENT_BINARY_DIR}/resources + COMMAND ${CMAKE_COMMAND} -E copy ${TEST_RESOURCE_SRC_PATHS} + ${CMAKE_CURRENT_BINARY_DIR}/resources/ + COMMAND_EXPAND_LISTS + COMMENT "Copy test resources to build directory" + DEPENDS ${TEST_RESOURCE_SRC_PATHS}) +add_custom_target( + copy_test_resources + COMMENT "Target for test resources" + DEPENDS ${TEST_RESOURCE_BUILD_PATHS}) + add_executable( test_${PROJECT_NAME} main.cpp src/default_config.cpp src/utils.cpp tests/asl_formatter.cpp + tests/commandline.cpp tests/config_arrays.cpp tests/config_maps.cpp tests/conversions.cpp @@ -29,15 +53,10 @@ add_executable( tests/update.cpp tests/validity_checks.cpp tests/virtual_config.cpp - tests/yaml_parsing.cpp) + tests/yaml_parsing.cpp + tests/yaml_utils.cpp) +add_dependencies(test_${PROJECT_NAME} copy_test_resources) target_include_directories(test_${PROJECT_NAME} PRIVATE include) target_link_libraries(test_${PROJECT_NAME} PRIVATE ${PROJECT_NAME} GTest::gtest_main) gtest_add_tests(TARGET test_${PROJECT_NAME}) - -if(${CONFIG_UTILS_INSTALL_TESTS}) - # note that ros uses libdir to handle finding executables by package, but this - # isn't an ideal install location normally - install(TARGETS test_${PROJECT_NAME} - RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}) -endif() diff --git a/config_utilities/test/resources/bar.yaml b/config_utilities/test/resources/bar.yaml new file mode 100644 index 0000000..8dafb88 --- /dev/null +++ b/config_utilities/test/resources/bar.yaml @@ -0,0 +1,3 @@ +a: 6.0 +b: [4] +d: world! diff --git a/config_utilities/test/resources/foo.yaml b/config_utilities/test/resources/foo.yaml new file mode 100644 index 0000000..0f353f0 --- /dev/null +++ b/config_utilities/test/resources/foo.yaml @@ -0,0 +1,4 @@ +--- +a: 5.0 +b: [1, 2, 3] +c: hello diff --git a/config_utilities/test/resources/invalid.yaml b/config_utilities/test/resources/invalid.yaml new file mode 100644 index 0000000..d516bc0 --- /dev/null +++ b/config_utilities/test/resources/invalid.yaml @@ -0,0 +1 @@ +invalid: {incomplete: dict diff --git a/config_utilities/test/tests/commandline.cpp b/config_utilities/test/tests/commandline.cpp new file mode 100644 index 0000000..a2631e3 --- /dev/null +++ b/config_utilities/test/tests/commandline.cpp @@ -0,0 +1,224 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/parsing/commandline.h" + +#include + +#include "config_utilities/test/utils.h" + +namespace config::test { +namespace { + +struct CliArgs { + struct Args { + int argc; + char** argv; + + std::string get_cmd() const { + std::stringstream ss; + for (int i = 0; i < argc; ++i) { + ss << argv[i]; + if (i < argc - 1) { + ss << " "; + } + } + + return ss.str(); + } + }; + + explicit CliArgs(const std::vector& args) : original_args(args) { + for (auto& str : original_args) { + arg_pointers.push_back(str.data()); + } + } + + Args get() { return {static_cast(arg_pointers.size()), arg_pointers.data()}; } + + std::vector original_args; + std::vector arg_pointers; +}; + +} // namespace + +TEST(Commandline, noInputArgs) { + CliArgs cli_args(std::vector{"some_command"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const YAML::Node expected; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, invalidFlags) { + CliArgs cli_args(std::vector{"some_command", "--config-utilities-file", "--config-utilities-yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const YAML::Node expected; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command --config-utilities-file --config-utilities-yaml"); +} + +TEST(Commandline, missingFile) { + CliArgs cli_args(std::vector{"some_command", "--config-utilities-file", "resources/missing.yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const YAML::Node expected; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, invalidFile) { + CliArgs cli_args(std::vector{"some_command", "--config-utilities-file", "resources/invalid.yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const YAML::Node expected; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, vaildFiles) { + CliArgs cli_args(std::vector{"some_command", + "--config-utilities-file", + "resources/foo.yaml", + "--config-utilities-file", + "resources/bar.yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const auto expected = YAML::Load(R"yaml( +a: 6.0 +b: [1, 2, 3, 4] +c: hello +d: world! + )yaml"); + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, fileOrderingCorrect) { + CliArgs cli_args(std::vector{"some_command", + "--config-utilities-file", + "resources/bar.yaml", + "--config-utilities-file", + "resources/foo.yaml"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, false); + const auto expected = YAML::Load(R"yaml( +a: 5.0 +b: [4, 1, 2, 3] +c: hello +d: world! + )yaml"); + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), + "some_command --config-utilities-file resources/bar.yaml --config-utilities-file resources/foo.yaml"); +} + +TEST(Commandline, mixedYamlAndFiles) { + // note that ROS can't escape correctly so args are broken by space + CliArgs cli_args(std::vector{"some_command", + "--config-utilities-file", + "resources/foo.yaml", + "--config-utilities-yaml", + "{a:", + "6.0,", + "b:", + "[4],", + "d:", + "world!}"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const auto expected = YAML::Load(R"yaml( +a: 6.0 +b: [1, 2, 3, 4] +c: hello +d: world! + )yaml"); + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, InvalidYaml) { + // note that ROS can't escape correctly so args are broken by space + CliArgs cli_args(std::vector{ + "some_command", "--config-utilities-file", "resources/foo.yaml", "--config-utilities-yaml", "{a:", "6.0,"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const auto expected = YAML::Load(R"yaml( +a: 5.0 +b: [1, 2, 3] +c: hello + )yaml"); + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command"); +} + +TEST(Commandline, NamespacedFile) { + // note that ROS can't escape correctly so args are broken by space + // NOTE(nathan) adds intermediate flags to also check arg removal + // NOTE(nathan) order should be preserved: bar.yaml should override a for inline yaml + CliArgs cli_args(std::vector{"some_command", + "--config-utilities-file", + "resources/foo.yaml@foo/other", + "--verbose=true", + "--config-utilities-yaml", + "{c:", + "6.0, a: 7.0}", + "--some-flag", + "--config-utilities-file", + "resources/bar.yaml@", + "--ros-args", + "-r", + "other_arg:=something"}); + auto args = cli_args.get(); + const auto node = internal::loadFromArguments(args.argc, args.argv, true); + const auto expected = YAML::Load(R"yaml( +foo: + other: + a: 5.0 + b: [1, 2, 3] + c: hello +c: 6.0 +a: 6.0 +b: [4] +d: world! + )yaml"); + std::cerr << node << std::endl; + expectEqual(expected, node); + EXPECT_EQ(args.get_cmd(), "some_command --verbose=true --some-flag --ros-args -r other_arg:=something"); +} + +} // namespace config::test diff --git a/config_utilities/test/tests/external_registry.cpp b/config_utilities/test/tests/external_registry.cpp index 2274463..cfa4890 100644 --- a/config_utilities/test/tests/external_registry.cpp +++ b/config_utilities/test/tests/external_registry.cpp @@ -38,6 +38,7 @@ #include #include "config_utilities/logging/log_to_stdout.h" +#include "config_utilities/settings.h" #include "config_utilities/test/external_types.h" #include "config_utilities/test/utils.h" @@ -49,6 +50,27 @@ struct LoggerGuard { std::shared_ptr logger; }; +struct SettingsGuard { + SettingsGuard() {} + ~SettingsGuard() { Settings().restoreDefaults(); } +}; + +TEST(ExternalRegistry, MoveableGuard) { + // guard is valid with at least one library + internal::LibraryGuard guard("some_library_path"); + EXPECT_TRUE(guard); + + // default constructor is invalid state + internal::LibraryGuard other = std::move(guard); + EXPECT_FALSE(guard); + EXPECT_TRUE(other); + + // move assignment should revert invalid/valid states + guard = std::move(other); + EXPECT_FALSE(other); + EXPECT_TRUE(guard); +} + TEST(ExternalRegistry, InstanceLifetimes) { auto plugin_lib = loadExternalFactories("./test_config_utilities_plugins"); auto unmanaged_logger = create("external_logger"); @@ -89,13 +111,25 @@ TEST(ExternalRegistry, InvalidFile) { EXPECT_EQ(logger_guard.logger->lastMessage().find("Unable to load library"), 0); } +TEST(ExternalRegistry, DisableLoading) { + config::test::SettingsGuard settings_guard; + config::Settings().allow_external_libraries = false; + + const auto guard = config::loadExternalFactories("./test_config_utilities_plugins"); + EXPECT_FALSE(guard); +} + } // namespace config::test // globally namespaced to check example compilation TEST(ExternalRegistry, ManagedInstance) { + config::test::SettingsGuard settings_guard; + config::Settings().print_external_allocations = true; + config::ManagedInstance talker; { // scope where external library is loaded - const auto guard = config::loadExternalFactories("./test_config_utilities_plugins"); + const std::vector to_load{"./test_config_utilities_plugins"}; + const auto guard = config::loadExternalFactories(to_load); talker = config::createManaged(config::create("external")); const auto view = talker.get(); ASSERT_TRUE(view); @@ -106,7 +140,8 @@ TEST(ExternalRegistry, ManagedInstance) { EXPECT_FALSE(view); { // scope where external library is loaded - const auto guard = config::loadExternalFactories("./test_config_utilities_plugins"); + const std::vector to_load{"./test_config_utilities_plugins"}; + const auto guard = config::loadExternalFactories(to_load); talker = config::createManaged(config::create("internal")); EXPECT_TRUE(talker); } // external library is unloaded after this point diff --git a/config_utilities/test/tests/yaml_parsing.cpp b/config_utilities/test/tests/yaml_parsing.cpp index c28657e..b1a1199 100644 --- a/config_utilities/test/tests/yaml_parsing.cpp +++ b/config_utilities/test/tests/yaml_parsing.cpp @@ -38,70 +38,12 @@ #include #include "config_utilities/config.h" -#include "config_utilities/internal/yaml_utils.h" #include "config_utilities/parsing/yaml.h" #include "config_utilities/test/default_config.h" #include "config_utilities/test/utils.h" namespace config::test { -YAML::Node createData() { - YAML::Node data; - data["a"]["b"]["c"] = 1; - data["a"]["b"]["d"] = "test"; - data["a"]["b"]["e"] = std::vector({1, 2, 3}); - data["a"]["b"]["f"] = std::map({{"1_str", 1}, {"2_str", 2}}); - data["a"]["g"] = 3; - return data; -} - -TEST(YamlParsing, lookupNamespace) { - YAML::Node data = createData(); - - expectEqual(data, data); - - YAML::Node data_1 = YAML::Clone(data); - data_1["a"]["b"]["c"] = 2; - EXPECT_FALSE(internal::isEqual(data, data_1)); - - YAML::Node data_2 = internal::lookupNamespace(data, ""); - // NOTE(lschmid): lookupNamespace returns a pointer, so this should be identity. - EXPECT_TRUE(data == data_2); - expectEqual(data, data_2); - - YAML::Node b = internal::lookupNamespace(data, "a/b"); - // NOTE(lschmid): lookupNamespace returns a pointer, so this should be identity. - EXPECT_TRUE(b == data["a"]["b"]); - expectEqual(b, data["a"]["b"]); - - YAML::Node b2 = internal::lookupNamespace(YAML::Clone(data), "a/b"); - - expectEqual(b2, data["a"]["b"]); - - YAML::Node c = internal::lookupNamespace(data, "a/b/c"); - EXPECT_TRUE(c.IsScalar()); - EXPECT_EQ(c.as(), 1); - - YAML::Node invalid = internal::lookupNamespace(data, "a/b/c/d"); - EXPECT_FALSE(invalid.IsDefined()); - EXPECT_FALSE(static_cast(invalid)); - - // Make sure the input node is not modified. - expectEqual(data, createData()); -} - -TEST(YamlParsing, moveDownNamespace) { - YAML::Node data = createData(); - - internal::moveDownNamespace(data, ""); - expectEqual(data, createData()); - - YAML::Node expected_data; - expected_data["a"]["b"]["c"] = createData(); - internal::moveDownNamespace(data, "a/b/c"); - expectEqual(data, expected_data); -} - TEST(YamlParsing, parsefromYaml) { DefaultConfig config; YAML::Node data = DefaultConfig::modifiedValues(); @@ -178,6 +120,26 @@ my_enum: "D" EXPECT_EQ(errors[4]->message(), "Name 'D' is out of bounds for enum with names ['A', 'B', 'C']"); } +TEST(YamlParsing, overflowConversionFailure) { + const auto node = YAML::Load(R"yaml({under: -1, over: 256})yaml"); + + { // values below [0, 255] cause errors + uint8_t value = 0; + std::string error; + EXPECT_FALSE(internal::YamlParser::fromYaml(node, "under", value, "", error)); + EXPECT_EQ(value, 0u); + EXPECT_EQ(error, "Value '-1' underflows storage min of '0'."); + } + + { // values above [0, 255] cause errors + uint8_t value = 0; + std::string error; + EXPECT_FALSE(internal::YamlParser::fromYaml(node, "over", value, "", error)); + EXPECT_EQ(value, 0u); + EXPECT_EQ(error, "Value '256' overflows storage max of '255'."); + } +} + TEST(YamlParsing, setValues) { YAML::Node data = DefaultConfig::modifiedValues(); DefaultConfig config; diff --git a/config_utilities/test/tests/yaml_utils.cpp b/config_utilities/test/tests/yaml_utils.cpp new file mode 100644 index 0000000..e80502c --- /dev/null +++ b/config_utilities/test/tests/yaml_utils.cpp @@ -0,0 +1,222 @@ +/** ----------------------------------------------------------------------------- + * Copyright (c) 2023 Massachusetts Institute of Technology. + * All Rights Reserved. + * + * AUTHORS: Lukas Schmid , Nathan Hughes + * AFFILIATION: MIT-SPARK Lab, Massachusetts Institute of Technology + * YEAR: 2023 + * LICENSE: BSD 3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * -------------------------------------------------------------------------- */ + +#include "config_utilities/internal/yaml_utils.h" + +#include + +#include + +#include "config_utilities/test/utils.h" + +namespace config::test { +namespace { +inline YAML::Node createData() { + YAML::Node data; + data["a"]["b"]["c"] = 1; + data["a"]["b"]["d"] = "test"; + data["a"]["b"]["e"] = std::vector({1, 2, 3}); + data["a"]["b"]["f"] = std::map({{"1_str", 1}, {"2_str", 2}}); + data["a"]["g"] = 3; + return data; +} + +inline YAML::Node doMerge(const YAML::Node& lhs, const YAML::Node& rhs, bool extend_lists = false) { + auto result = YAML::Clone(lhs); + internal::mergeYamlNodes(result, rhs, extend_lists); + return result; +} + +} // namespace + +TEST(YamlUtils, mergeYamlNodes) { + auto node_a = YAML::Load(R"""(root: {a: 1, b: 2})"""); + const auto node_b = YAML::Load(R"""(root: {a: 1, c: 3})"""); + internal::mergeYamlNodes(node_a, node_b); + + const auto result = doMerge(node_a, node_b); + const auto expected = YAML::Load(R"""(root: {a: 1, b: 2, c: 3})"""); + expectEqual(result, expected); +} + +TEST(YamlUtils, mergeYamlNodesInvalidKey) { + const auto node_a = YAML::Load(R"""(root: {a: 1, b: 2})"""); + const auto node_b = YAML::Load(R"""(? [root, other]: {a: 1, c: 3})"""); + + const auto result = doMerge(node_a, node_b); + const auto expected = YAML::Load(R"""(root: {a: 1, b: 2})"""); + expectEqual(result, expected); +} + +TEST(YamlUtils, mergeYamlNodesExtend) { + // NOTE(nathan) structured to require appending at different levels of recursion + const auto node_a = YAML::Load(R"""( +root: + a: 1 + b: [2] + c: + foo: [1, 2, 3] + bar: [{1: 2, 3: 4}, {5, 6}] +other: [1, 2] +)"""); + const auto node_b = YAML::Load(R"""( +root: + a: 1 + b: [4] + c: + bar: [7] + d: 3.0 +other: [3, 4, 5] +)"""); + + { // without extend, lists should override + auto result = doMerge(node_a, node_b); + const auto expected = YAML::Load(R"""( +root: + a: 1 + b: [4] + c: + foo: [1, 2, 3] + bar: [7] + d: 3.0 +other: [3, 4, 5] +)"""); + expectEqual(result, expected); + } + + { // with extend, lists should append + auto result = doMerge(node_a, node_b, true); + const auto expected = YAML::Load(R"""( +root: + a: 1 + b: [2, 4] + c: + foo: [1, 2, 3] + bar: [{1: 2, 3: 4}, {5, 6}, 7] + d: 3.0 +other: [1, 2, 3, 4, 5] +)"""); + expectEqual(result, expected); + } +} + +TEST(YamlUtils, isEqual) { + { // different node types are inequal + const auto node_a = YAML::Load(R"""(5)"""); + const auto node_b = YAML::Load(R"""([5])"""); + EXPECT_TRUE(internal::isEqual(node_a, YAML::Clone(node_a))); + EXPECT_FALSE(internal::isEqual(node_a, node_b)); + } + + { // sequences work as expected + const auto node_a = YAML::Load(R"""([1, 2, 3, 4, 5])"""); + const auto node_b = YAML::Load(R"""([1, 2, 2, 4, 5])"""); + const auto node_c = YAML::Load(R"""([1, 2, 2, 4])"""); + EXPECT_TRUE(internal::isEqual(node_a, YAML::Clone(node_a))); + EXPECT_FALSE(internal::isEqual(node_a, node_b)); + EXPECT_FALSE(internal::isEqual(node_a, node_c)); + EXPECT_FALSE(internal::isEqual(node_b, node_c)); + } + + { // maps work as expected + const auto node_a = YAML::Load(R"""({a: 1, b: 2, c: 3})"""); + const auto node_b = YAML::Load(R"""({a: 1, b: 1, c: 3})"""); + const auto node_c = YAML::Load(R"""({a: 1, d: 1, c: 3})"""); + const auto node_d = YAML::Load(R"""({a: 1, b: 2})"""); + EXPECT_TRUE(internal::isEqual(node_a, YAML::Clone(node_a))); + EXPECT_FALSE(internal::isEqual(node_a, node_b)); + EXPECT_FALSE(internal::isEqual(node_b, node_c)); + EXPECT_FALSE(internal::isEqual(node_a, node_c)); + EXPECT_FALSE(internal::isEqual(node_b, node_d)); + } + + { // null nodes are equal + const auto node_a = YAML::Node(); + const auto node_b = YAML::Node(); + const auto node_c = YAML::Load(R"""({a: 1, d: 1, c: 3})"""); + EXPECT_TRUE(internal::isEqual(node_a, node_b)); + EXPECT_FALSE(internal::isEqual(node_a, node_c)); + EXPECT_FALSE(internal::isEqual(node_b, node_c)); + } +} + +TEST(YamlUtils, lookupNamespace) { + YAML::Node data = createData(); + + expectEqual(data, data); + + YAML::Node data_1 = YAML::Clone(data); + data_1["a"]["b"]["c"] = 2; + EXPECT_FALSE(internal::isEqual(data, data_1)); + + YAML::Node data_2 = internal::lookupNamespace(data, ""); + // NOTE(lschmid): lookupNamespace returns a pointer, so this should be identity. + EXPECT_TRUE(data == data_2); + expectEqual(data, data_2); + + YAML::Node b = internal::lookupNamespace(data, "a/b"); + // NOTE(lschmid): lookupNamespace returns a pointer, so this should be identity. + EXPECT_TRUE(b == data["a"]["b"]); + expectEqual(b, data["a"]["b"]); + + YAML::Node b2 = internal::lookupNamespace(YAML::Clone(data), "a/b"); + + expectEqual(b2, data["a"]["b"]); + + YAML::Node c = internal::lookupNamespace(data, "a/b/c"); + EXPECT_TRUE(c.IsScalar()); + EXPECT_EQ(c.as(), 1); + + YAML::Node invalid = internal::lookupNamespace(data, "a/b/c/d"); + EXPECT_FALSE(invalid.IsDefined()); + EXPECT_FALSE(static_cast(invalid)); + + // Make sure the input node is not modified. + expectEqual(data, createData()); +} + +TEST(YamlUtils, moveDownNamespace) { + YAML::Node data = createData(); + + internal::moveDownNamespace(data, ""); + expectEqual(data, createData()); + + YAML::Node expected_data; + expected_data["a"]["b"]["c"] = createData(); + internal::moveDownNamespace(data, "a/b/c"); + expectEqual(data, expected_data); +} + +} // namespace config::test diff --git a/docs/Parsing.md b/docs/Parsing.md index 0aea783..1692f8a 100644 --- a/docs/Parsing.md +++ b/docs/Parsing.md @@ -4,6 +4,8 @@ This tutorial explains how to create configs and other objects from source data. **Contents:** - [Parse from yaml](#parse-from-yaml) - [Parse from ROS](#parse-from-ros) +- [Parse from the command line](#parse-from-the-command-line) +- [Parse via global context](#parse-via-global-context) ## Parse from yaml @@ -88,3 +90,78 @@ To use [factory creation with configs](Factories.md#creating-objects-with-indivi std::unique_ptr object = createFromRosMyBase>(nh); std::unique_ptr object = createFromRosWithNamespace(nh, ns); ``` + +## Parse from the command line + +It is also possible to use the same interfaces as in the yaml or ROS case but via aggregate YAML read from the command line. To use it, include `parsing/command_line.h`. +`config_utilities` supports parsing two command line flags: + - `--config-utilities-file SOME_FILE_PATH`: Specify a file to load YAML from. + - `--config-utilities-yaml SOME_ARBITRARY_YAML`: Specify YAML directly from the command line. + +> **✅ Supports**
+> Note that the `--config-utilities-file` flag allows for a namespace (i.e., `some/custom/ns`) to apply to the file globally. This is specified as `--config-utilities-file SOME_FILE@some/custom/ns`. + +Both command line flags can be specified as many times as needed. +When aggregating the YAML from the command line, the various flags are merged left to right (where conflicting keys from the last specified flag take precedence) and any sequences are appended together. +For those familiar with how the ROS parameter server works, this is the same behavior. +Please also note that the `--config-utilities-yaml` currently accepts multiple space-delimited tokens (because the ROS2 launch file infrastructure does not currently correctly handle escaped substitutions), so +``` +some_command --config-utilities-yaml '{my: {cool: config}}' --config-utilities-file some_file.yaml +``` +and +``` +some_command --config-utilities-yaml {my: {cool: config}} --config-utilities-file some_file.yaml +``` +will result in the same behavior (that the resulting parsed YAML will be `{my: {cool: config}}` merged with the contents of `some_file.yaml`). + +Parsing directly from the command line takes one of three forms: +```c++ +int main(int argc, char** argv) { + // Instantiate config struct directly. + MyConfig my_config = config::fromCLI(argc, argv, "optional/namespace"); + + // Factory-based instantiation (where base_args... are arguments to the object constructor) + const auto object_1 = config::createFromCLI(argc, argv, base_args...); + + // Factory-based instantiation with namespace (where base_args... are arguments to the object constructor) + const auto object_2 = config::createFromCLI(argc, argv, "optional/namespace", base_args...); +} +``` + +# Parse via global context + +Usually the command line arguments or parsed YAML are not globally available to every part of an executable. +Similar to the ROS1 parameter server (and access to the parameter server by `ros::NodeHandle`), we provide a global `config::internal::Context` object (included via `parsing/context.h`) that handles tracking parsed YAML. +This `config::internal::Context` is not intended to be manipulated directly. +Instead, you should use one of the following methods: +```cpp +int main(int argc, char** argv) { + // pushes config-utilities specific flags to the end of argv and decrements argc so that it + // looks like the command was run without any config-utilities specific flags + const bool remove_config_utils_args = true; + config::initContext(argc, argv, remove_config_utils_args); + + // adds "{some: {namespace: {a: 5}}}" to the global context + config::pushToContext(YAML::Load("{a: 5}", "some/namespace")); + + // saves the loaded context + std::ofstream out("config.yaml"); + out << config::contextToYaml(); + + // clears any loaded context + config::clearContext(); +} +``` +Please note that the global context is not threadsafe. + +This object alllows instantiating configs and objects by the same three forms as the other parsing methods: +```c++ +// Instantiate config struct directly. +MyConfig my_config = config::fromContext("optional/namespace"); + +// Factory-based instantiation (where base_args... are arguments to the object constructor) +const auto object_1 = config::createFromContext(base_args...); + +// Factory-based instantiation with namespace (where base_args... are arguments to the object constructor) +const auto object_2 = config::createFromContext("optional/namespace", base_args...); +``` diff --git a/docs/README.md b/docs/README.md index daa6297..7625795 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,8 @@ The following tutorials will guide you through functionalities of `config_utilit 3. [**Parsing configs from data sources**](Parsing.md) - [Parse from yaml](Parsing.md#parse-from-yaml) - [Parse from ROS](Parsing.md#parse-from-ros) + - [Parse from the command line](Parsing.md#parse-from-the-command-line) + - [Parse via global context](Parsing.md#parse-via-global-context) 4. [**Handling complex configs or types**](Types.md) - [Sub-configs](Types.md#sub-configs)