From 0449a3b73d972a99524d90e63c6a132827613ecb Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Fri, 7 Feb 2025 14:14:31 +0100 Subject: [PATCH 01/12] [Prop] Introduce nntrainer::PathProperty It is common to have an configuration property that includes paths, introduce new property kind; Its implementation is based on portable library. This frees us from path parsing, handing in cross-platform manner. Also it provides basic file operations, like creating directoris. Signed-off-by: Piotr Rak --- meson.build | 3 + nntrainer/meson.build | 3 +- nntrainer/utils/base_properties.cpp | 90 +++++++++++++++++++++++++++++ nntrainer/utils/base_properties.h | 45 +++++++++++++++ nntrainer/utils/util_func.h | 1 + 5 files changed, 141 insertions(+), 1 deletion(-) diff --git a/meson.build b/meson.build index acde363bbc..060d87e7d3 100644 --- a/meson.build +++ b/meson.build @@ -26,6 +26,9 @@ extra_defines = ['-DMIN_CPP_VERSION=201703L'] cc = meson.get_compiler('c') cxx = meson.get_compiler('cpp') +# Older libc++ has in seperate library +opt_stdcpp_fs_dep = cxx.find_library('stdc++fs', required: false) + if get_option('platform') == 'tizen' # Pass __TIZEN__ to the compiler add_project_arguments('-D__TIZEN__=1', language:['c','cpp']) diff --git a/nntrainer/meson.build b/nntrainer/meson.build index 68f1021018..4a4d2cc2b5 100644 --- a/nntrainer/meson.build +++ b/nntrainer/meson.build @@ -27,7 +27,8 @@ nntrainer_base_deps=[ libm_dep, libdl_dep, thread_dep, - openmp_dep + openmp_dep, + opt_stdcpp_fs_dep ] if get_option('platform') == 'tizen' diff --git a/nntrainer/utils/base_properties.cpp b/nntrainer/utils/base_properties.cpp index 30e0cc3347..ebd33a2317 100644 --- a/nntrainer/utils/base_properties.cpp +++ b/nntrainer/utils/base_properties.cpp @@ -11,11 +11,49 @@ */ #include +#include #include #include #include +#include #include +#if !defined(_WIN32) +#include +#endif + +namespace fs = std::filesystem; + +namespace { + +/** + * @brief constructs system error_code from POSIX errno + */ +[[maybe_unused]] auto make_system_error_code() { + return std::error_code{errno, std::system_category()}; +} + +bool isFileReadAccessisble(fs::path p, std::error_code &ec) { +#if defined(_POSIX_VERSION) + // Check if we have read permissions to path pointing this file + auto r = ::access(p.c_str(), R_OK); + if (r == 0) { + ec = std::error_code{}; + return true; + } + + ec = make_system_error_code(); + + return false; +#else + ec = std::error_code{}; + // Unless it is POSIX, best bet is to try it + std::ifstream file(p, std::ios::binary | std::ios::ate); + return file.good(); +#endif +} +} // namespace + namespace nntrainer { bool PositiveIntegerProperty::isValid(const unsigned int &value) const { return value > 0; @@ -33,6 +71,18 @@ std::string str_converter::from_string( return value; } +template <> +std::string +str_converter::to_string(const fs::path &value) { + return value; +} + +template <> +fs::path +str_converter::from_string(const std::string &value) { + return fs::path{value}; +} + template <> std::string str_converter::to_string(const bool &value) { return value ? "true" : "false"; @@ -144,4 +194,44 @@ TensorDim str_converter::from_string( return target; } +PathProperty::~PathProperty() = default; + +bool PathProperty::isRegularFile(const fs::path &v) noexcept { + // Reject empty and non-existing paths + { + std::error_code ec; + if (v.empty() || !exists(v, ec)) + return false; + + if (ec) + return false; + } + + // Check if it is a path to regular file + { + std::error_code ec; + auto real_path = is_symlink(v)? read_symlink(v, ec) : v; + + if (ec) + return false; + + if (!is_regular_file(real_path, ec)) + return false; + + if (ec) + return false; + } + + return true; +} + +bool PathProperty::isReadAccessible(const fs::path &v) noexcept { + std::error_code ec; + + if (!isFileReadAccessisble(v, ec)) + return false; + + return !ec; +} + } // namespace nntrainer diff --git a/nntrainer/utils/base_properties.h b/nntrainer/utils/base_properties.h index ceee6bef8e..4b8a769db6 100644 --- a/nntrainer/utils/base_properties.h +++ b/nntrainer/utils/base_properties.h @@ -13,6 +13,7 @@ #define __BASE_PROPERTIES_H__ #include +#include #include #include #include @@ -116,6 +117,12 @@ struct double_prop_tag {}; */ struct str_prop_tag {}; +/** + * @brief property is treated as a path + * + */ +struct path_prop_tag {}; + /** * @brief property is treated as boolean * @@ -266,6 +273,29 @@ template class Property { std::unique_ptr value; /**< underlying data */ }; +/** + * @brief The filesystem path property + */ +struct PathProperty : Property { + using base = Property; + + using base::base; + + using prop_tag = path_prop_tag; + + explicit PathProperty(const std::string &str) : base(str) {} + + virtual bool isValid(const std::filesystem::path &value) const override { + return true; + }; + + virtual ~PathProperty() override; + +protected: + static bool isRegularFile(const std::filesystem::path &) noexcept; + static bool isReadAccessible(const std::filesystem::path &) noexcept; +}; + /** * @brief enum property * @@ -461,6 +491,21 @@ template <> std::string str_converter::from_string(const std::string &value); +/** + * @copydoc template struct str_converter + */ +template <> +std::string str_converter::to_string( + const std::filesystem::path &value); + +/** + * @copydoc template struct str_converter + */ +template <> +std::filesystem::path +str_converter::from_string( + const std::string &value); + /** * @copydoc template struct str_converter */ diff --git a/nntrainer/utils/util_func.h b/nntrainer/utils/util_func.h index c041cabeda..8d26790ff3 100644 --- a/nntrainer/utils/util_func.h +++ b/nntrainer/utils/util_func.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include From b52efcfc3c221dc697de726fbd906a0f4088fcee Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Tue, 11 Feb 2025 15:42:38 +0100 Subject: [PATCH 02/12] [Prop] Add nntrainer::ModelPathProperty Specialized helper for file model path. Its main purpouse is to ensure file path meets criterions of given type model files. It covers PropsTflModelPath (file readability and correct file extension), while extending it to set of possible file extensions. Signed-off-by: Piotr Rak --- nntrainer/utils/base_properties.h | 122 ++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/nntrainer/utils/base_properties.h b/nntrainer/utils/base_properties.h index 4b8a769db6..45e15e9d26 100644 --- a/nntrainer/utils/base_properties.h +++ b/nntrainer/utils/base_properties.h @@ -12,12 +12,15 @@ #ifndef __BASE_PROPERTIES_H__ #define __BASE_PROPERTIES_H__ +#include #include #include #include #include +#include #include #include +#include #include #include @@ -765,6 +768,125 @@ class TensorFormat final : public EnumProperty { }; }; +/** + * @class ICaseLexOrder + * + * @brief The Lexicograpthic order strings disregarding case of the characters. + */ +struct ICaseLexOrder final { + /** + * @brief case insensitive character compare + * + * @return true if @param l is less than @param l + */ + constexpr static bool compare(unsigned char l, unsigned char r) noexcept { + return std::less{}(std::tolower(l), std::tolower(r)); + } + + /** + * @brief case insensitive character compare + * + * @return true if @param l is less than @param l + */ + constexpr bool operator()(unsigned char l, unsigned char r) const noexcept { + return ICaseLexOrder::compare(l, r); + } + + /** + * @brief case insensitive character compare + * + * @return true if @param l is less than @param l + */ + static bool compare(std::string_view l, std::string_view r) noexcept { + return std::lexicographical_compare(l.begin(), l.end(), r.begin(), r.end(), + ICaseLexOrder{}); + } + + /** + * @brief case insensitive character compare + * + * @return true if @param l is less than @param l + */ + bool operator()(std::string_view l, std::string_view r) const noexcept { + return ICaseLexOrder::compare(l, r); + } +}; + +/** + * @brief case-insensitive set of string views + */ +using ICaseStringSet = std::set; + +/** + * @brief Declares allowed model format extensions + * + * To specify allowed file extensions it is required to specialize: + * + * @code{.cpp} + * struct my_model_tag {}; + * + * template <> AllowedModelFormatExtSet::extensions = { + * "file_ext1"sv, "file_ext2"sv, // etc... + * }; + * + * // Those asserts should hold, given files: + * // - 'file.fILe_eXT1' + * // - '/bananas/file.FILE_ext2' + * // exist in the file-system and read-accessible regular files or symlinks + * // to thereof. + * { + * assert(ModelPathProperty{}.isValid("file.fILe_eXT1")); + * assert(ModelPathProperty{}.isValid("/bananas/file.FILE_ext2")); + * } + * + * @endcode + */ +template class AllowedModelFormatExtSet { + + static const ICaseStringSet extensions; + +public: + /** + * @brief check if the file extension (case-insensetive) is in the set + */ + constexpr static bool has(std::string_view ext) noexcept { + return extensions.find(ext) != extensions.end(); // c++20 std::set::contains + } +}; + +/** + * @brief Trait returing if model file extension is is allowed/supported. + */ +template struct model_extension_traits { + constexpr static bool isFileExtAllowed(std::string_view file_ext) noexcept { + return AllowedModelFormatExtSet::has(file_ext); + } +}; + +template class ModelPathProperty : public PathProperty { + +public: + using prop_tag = path_prop_tag; + + bool isValid(const std::filesystem::path &v) const override { + + if (!PathProperty::isRegularFile(v)) + return false; + + // Checked if file has allowed supported file extension + std::string ext = v.extension(); + + if (!model_extension_traits::isFileExtAllowed(ext)) + return false; + + // Reject path we don't have read access permission + if (!PathProperty::isReadAccessible(v)) + return false; + + return true; + } +}; + // /** // * @brief trainable property, use this to set and check how if certain layer // is From 0b7ac79fe6de0c2b2f6c4fd5ff25150d8a2bd974 Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Tue, 11 Feb 2025 20:33:09 +0100 Subject: [PATCH 03/12] [TEST] Enable unittest testsuite build without tensorflow-lite This allows to build test-suite w/o tensorflow-lite. Probably can be dropped when we'll stop using python-env provided OpenVINO. --- test/meson.build | 2 ++ test/unittest/compiler/meson.build | 5 ++++- test/unittest/integration_tests/meson.build | 7 +++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/meson.build b/test/meson.build index b3d99e92cf..4cc360bce4 100644 --- a/test/meson.build +++ b/test/meson.build @@ -15,6 +15,7 @@ nntrainer_testutil_dep = declare_dependency( link_with: nntrainer_testutil_lib, include_directories: nntrainer_test_inc ) +gtest_subproj_main_dep = dependency('gtest_main', required: false) nntrainer_test_deps = [ gmock_dep, @@ -27,6 +28,7 @@ nntrainer_test_deps = [ nntrainer_test_main_deps = [ gmock_dep, gtest_main_dep, + gtest_subproj_main_dep, nntrainer_dep, nntrainer_testutil_dep ] diff --git a/test/unittest/compiler/meson.build b/test/unittest/compiler/meson.build index cf9a864577..6e028fe157 100644 --- a/test/unittest/compiler/meson.build +++ b/test/unittest/compiler/meson.build @@ -4,10 +4,13 @@ test_target = [ 'compiler_test_util.cpp', 'unittest_compiler.cpp', 'unittest_interpreter.cpp', - 'unittest_tflite_export.cpp', 'unittest_realizer.cpp', ] +if get_option('enable-tflite-interpreter') + test_target += 'unittest_tflite_export.cpp' +endif + exe = executable( test_name, test_target, diff --git a/test/unittest/integration_tests/meson.build b/test/unittest/integration_tests/meson.build index 8071a860fc..50862610b7 100644 --- a/test/unittest/integration_tests/meson.build +++ b/test/unittest/integration_tests/meson.build @@ -3,9 +3,12 @@ mixed_precision_test_name = 'integration_test_mixed_precision' test_target = [ 'integration_tests.cpp', - 'integration_test_loss.cpp', ] +if (get_option('enable-tflite-interpreter')) + test_target += 'integration_test_loss.cpp' +endif + mixed_precision_targets = [ model_util_path / 'models_test_utils.cpp', 'integration_test_mixed_precision.cpp', @@ -26,7 +29,7 @@ exe = executable( test_target, include_directories: [include_directories('.'), model_util_include_dir], dependencies: [ - nntrainer_test_main_deps, + nntrainer_test_deps, nntrainer_dep, nntrainer_ccapi_dep, ], From ffc3b341b39142c1c46d6ba4711372a511f294a0 Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Tue, 11 Feb 2025 14:27:18 +0100 Subject: [PATCH 04/12] [TEST] nntrainer::PathProperty bananas tests Adds sanity tests for path property. --- test/unittest/unittest_base_properties.cpp | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/unittest/unittest_base_properties.cpp b/test/unittest/unittest_base_properties.cpp index e25c97386a..7c1e25daa2 100644 --- a/test/unittest/unittest_base_properties.cpp +++ b/test/unittest/unittest_base_properties.cpp @@ -24,6 +24,8 @@ namespace { /**< define a property for testing */ +namespace fs = std::filesystem; + /** * @brief banana property tag for example * @@ -60,6 +62,22 @@ class QualityOfBanana : public nntrainer::Property { } }; +struct PathToNowhere : public nntrainer::PathProperty { + PathToNowhere() = default; + PathToNowhere(const fs::path &path); + static constexpr const char *key = "path_to_nowhere"; + + bool isValid(const fs::path &v) const override { return !exists(v); } +}; + +struct PathToBananaFarm : public nntrainer::PathProperty { + PathToBananaFarm() = default; + PathToBananaFarm(const fs::path &path); + static constexpr const char *key = "path_to_farm"; + + bool isValid(const fs::path &v) const override { return exists(v); } +}; + /** * @brief Property example to be used as a bool * @@ -271,6 +289,24 @@ TEST(BasicProperty, valid_p) { EXPECT_FLOAT_EQ(q.get(), 1.3245f); } + { /** set -> get / from_string, fs::path (doesn't exist) */ + const auto pth_banana = fs::path("/golden-BANANA-farm"); + + EXPECT_FALSE(exists(pth_banana)); + PathToNowhere p; + nntrainer::from_string("/golden-BANANA-farm", p); + EXPECT_EQ(pth_banana, p.get()); + } + + { /** set -> get / from_string, fs::path (path exists) */ + const auto cwd = + fs::current_path(); // We can be certain that process has cwd + EXPECT_TRUE(exists(cwd)); + PathToBananaFarm p; + p.set(cwd); + EXPECT_EQ(nntrainer::to_string(p), fs::current_path()); + } + { /**< enum type test from_string -> get */ BananaType t; nntrainer::from_string("CAVENDISH", t); From 8297c5b5ab5d77501fb5d599c03e94155995c227 Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Tue, 11 Feb 2025 21:05:23 +0100 Subject: [PATCH 05/12] [WIP] Workarounds missing nnstreamer on ubuntu 24.04 LTS Fixes build with gtest as subproject. --- meson.build | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/meson.build b/meson.build index 060d87e7d3..9736effc29 100644 --- a/meson.build +++ b/meson.build @@ -21,6 +21,10 @@ add_project_arguments('-DVERSION_MAJOR=' + nntrainer_version_split[0], language: add_project_arguments('-DVERSION_MINOR=' + nntrainer_version_split[1], language: ['c', 'cpp']) add_project_arguments('-DVERSION_MICRO=' + nntrainer_version_split[2], language: ['c', 'cpp']) +if get_option('enable-test')# and get_option('enable-openvino-backbone').enabled() + gtest_proj = subproject('gtest') +endif + extra_defines = ['-DMIN_CPP_VERSION=201703L'] cc = meson.get_compiler('c') @@ -496,7 +500,7 @@ if get_option('enable-app') endif endif -if get_option('platform') != 'android' +if get_option('platform') != 'android' and get_option('enable-nnstreamer-backbone') nnstreamer_dep = dependency('nnstreamer') message('building nnstreamer') subdir('nnstreamer') From b1bdca0b424ab8fdc308fe6c0f00091be1707f1c Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Tue, 11 Feb 2025 23:24:08 +0100 Subject: [PATCH 06/12] [Prop] Switch nntrainer::props::FilePath to PathProperty This avoids opening file stream. Signed-off-by: Piotr Rak --- nntrainer/layers/common_properties.cpp | 18 ++++++++++-------- nntrainer/layers/common_properties.h | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/nntrainer/layers/common_properties.cpp b/nntrainer/layers/common_properties.cpp index 3ce4208d78..7980f295c7 100644 --- a/nntrainer/layers/common_properties.cpp +++ b/nntrainer/layers/common_properties.cpp @@ -24,6 +24,8 @@ #include #include +namespace fs = std::filesystem; + namespace nntrainer { namespace props { @@ -56,18 +58,18 @@ void RandomTranslate::set(const float &value) { Property::set(std::abs(value)); } -bool FilePath::isValid(const std::string &v) const { - std::ifstream file(v, std::ios::binary | std::ios::ate); - return file.good(); +bool FilePath::isValid(const fs::path &v) const { + const bool regular = PathProperty::isRegularFile(v); + const bool is_readable = PathProperty::isReadAccessible(v); + return regular && is_readable; } -void FilePath::set(const std::string &v) { - Property::set(v); - std::ifstream file(v, std::ios::binary | std::ios::ate); - cached_pos_size = file.tellg(); +void FilePath::set(const fs::path &v) { + PathProperty::set(v); + cached_pos_size = fs::file_size(v); } -std::ifstream::pos_type FilePath::file_size() { return cached_pos_size; } +std::uintmax_t FilePath::file_size() const { return cached_pos_size; } bool LossScaleForMixed::isValid(const float &value) const { return (value != 0); diff --git a/nntrainer/layers/common_properties.h b/nntrainer/layers/common_properties.h index 62440a5849..9ca6484665 100644 --- a/nntrainer/layers/common_properties.h +++ b/nntrainer/layers/common_properties.h @@ -664,21 +664,22 @@ class RandomTranslate : public nntrainer::Property { * @brief Props containing file path value * */ -class FilePath : public Property { +class FilePath : public PathProperty { public: /** * @brief Construct a new File Path object */ - FilePath() : Property() {} + FilePath() : PathProperty() {} /** * @brief Construct a new File Path object * * @param path path to set */ - FilePath(const std::string &path) { set(path); } + explicit FilePath(const std::string &path) { set(path); } + FilePath(const std::filesystem::path &path) { set(path); } static constexpr const char *key = "path"; /**< unique key to access */ - using prop_tag = str_prop_tag; /**< property type */ + using prop_tag = path_prop_tag; /**< property type */ /** * @brief check if given value is valid @@ -686,24 +687,24 @@ class FilePath : public Property { * @param v value to check * @return bool true if valid */ - bool isValid(const std::string &v) const override; + bool isValid(const std::filesystem::path &v) const override; /** * @brief setter * * @param v value to set */ - void set(const std::string &v) override; + void set(const std::filesystem::path &v) override; /** * @brief return file size * - * @return std::ifstream::pos_type size of the file + * @return std::uintmax_t size of the file */ - std::ifstream::pos_type file_size(); + std::uintmax_t file_size() const; private: - std::ifstream::pos_type cached_pos_size; + std::uintmax_t cached_pos_size; }; /** From 5fe660bd446bc628c54fd74171158cb62e17f4c6 Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Wed, 12 Feb 2025 00:01:41 +0100 Subject: [PATCH 07/12] [Prop] Avoid caching file_size() for props::FilePath That should be cheap enough using stat() or alike underneath. Signed-off-by: Piotr Rak --- nntrainer/layers/common_properties.cpp | 3 +-- nntrainer/layers/common_properties.h | 3 --- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/nntrainer/layers/common_properties.cpp b/nntrainer/layers/common_properties.cpp index 7980f295c7..5fca0d7caf 100644 --- a/nntrainer/layers/common_properties.cpp +++ b/nntrainer/layers/common_properties.cpp @@ -66,10 +66,9 @@ bool FilePath::isValid(const fs::path &v) const { void FilePath::set(const fs::path &v) { PathProperty::set(v); - cached_pos_size = fs::file_size(v); } -std::uintmax_t FilePath::file_size() const { return cached_pos_size; } +std::uintmax_t FilePath::file_size() const { return fs::file_size(*this); } bool LossScaleForMixed::isValid(const float &value) const { return (value != 0); diff --git a/nntrainer/layers/common_properties.h b/nntrainer/layers/common_properties.h index 9ca6484665..6e39407b1e 100644 --- a/nntrainer/layers/common_properties.h +++ b/nntrainer/layers/common_properties.h @@ -702,9 +702,6 @@ class FilePath : public PathProperty { * @return std::uintmax_t size of the file */ std::uintmax_t file_size() const; - -private: - std::uintmax_t cached_pos_size; }; /** From 02546c00618f9d1fd05bb46a889fdff3dca4a7ce Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Wed, 12 Feb 2025 00:09:20 +0100 Subject: [PATCH 08/12] [BUG?] Rework cast for file_size in RawFileDataProducer This had potential to overflow for files larger than 4GB (2^32)B This days it is bit small. This change is NOT the real fix of this potential issue, but makes it (sample_size * RawFileDataProducer::pixel_size) times less likely. The return type of this function still holds just 2^32-1. Signed-off-by: Piotr Rak --- nntrainer/dataset/raw_file_data_producer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nntrainer/dataset/raw_file_data_producer.cpp b/nntrainer/dataset/raw_file_data_producer.cpp index e26b3f6fb2..69ae7f262d 100644 --- a/nntrainer/dataset/raw_file_data_producer.cpp +++ b/nntrainer/dataset/raw_file_data_producer.cpp @@ -109,8 +109,8 @@ RawFileDataProducer::size(const std::vector &input_dims, // << " Given file does not align with the given sample size, sample size: " // << sample_size << " file_size: " << file_size; - return static_cast(file_size) / - (sample_size * RawFileDataProducer::pixel_size); + return static_cast( + file_size / (sample_size * RawFileDataProducer::pixel_size)); } void RawFileDataProducer::exportTo( From dfdefb1464d18c9842e9e6cea5f5bdf9a9ff07df Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Wed, 12 Feb 2025 01:05:30 +0100 Subject: [PATCH 09/12] [Props] Rework DirPath and simply FilePath common properties. Signed-off-by: Piotr Rak --- nntrainer/layers/common_properties.cpp | 12 +---- nntrainer/layers/common_properties.h | 43 +++++++++-------- nntrainer/utils/base_properties.cpp | 64 ++++++++++++++++---------- nntrainer/utils/base_properties.h | 1 + 4 files changed, 63 insertions(+), 57 deletions(-) diff --git a/nntrainer/layers/common_properties.cpp b/nntrainer/layers/common_properties.cpp index 5fca0d7caf..23a55eb75d 100644 --- a/nntrainer/layers/common_properties.cpp +++ b/nntrainer/layers/common_properties.cpp @@ -20,7 +20,6 @@ #include #include -#include #include #include @@ -64,23 +63,16 @@ bool FilePath::isValid(const fs::path &v) const { return regular && is_readable; } -void FilePath::set(const fs::path &v) { - PathProperty::set(v); -} - std::uintmax_t FilePath::file_size() const { return fs::file_size(*this); } bool LossScaleForMixed::isValid(const float &value) const { return (value != 0); } -bool DirPath::isValid(const std::string &v) const { - struct stat dir; - return (stat(v.c_str(), &dir) == 0); +bool DirPath::isValid(const fs::path &v) const { + return PathProperty::isDirectory(v); } -void DirPath::set(const std::string &v) { Property::set(v); } - ReturnSequences::ReturnSequences(bool value) { set(value); } Bidirectional::Bidirectional(bool value) { set(value); } diff --git a/nntrainer/layers/common_properties.h b/nntrainer/layers/common_properties.h index 6e39407b1e..22ad796ed5 100644 --- a/nntrainer/layers/common_properties.h +++ b/nntrainer/layers/common_properties.h @@ -669,15 +669,21 @@ class FilePath : public PathProperty { /** * @brief Construct a new File Path object */ - FilePath() : PathProperty() {} + FilePath() = default; /** * @brief Construct a new File Path object * * @param path path to set */ - explicit FilePath(const std::string &path) { set(path); } - FilePath(const std::filesystem::path &path) { set(path); } + explicit FilePath(const std::string &path) : PathProperty(path) {} + + /** + * @brief Construct a new File Path object + * + * @param path path to set + */ + explicit FilePath(const std::filesystem::path &path) : PathProperty(path) {} static constexpr const char *key = "path"; /**< unique key to access */ using prop_tag = path_prop_tag; /**< property type */ @@ -689,13 +695,6 @@ class FilePath : public PathProperty { */ bool isValid(const std::filesystem::path &v) const override; - /** - * @brief setter - * - * @param v value to set - */ - void set(const std::filesystem::path &v) override; - /** * @brief return file size * @@ -708,36 +707,36 @@ class FilePath : public PathProperty { * @brief Props containing directory path value * */ -class DirPath : public Property { +class DirPath : public PathProperty { public: /** * @brief Construct a new Dir Path object */ - DirPath() : Property() {} + DirPath() = default; /** * @brief Construct a new Dir Path object * * @param path path to set */ - DirPath(const std::string &path) { set(path); } - static constexpr const char *key = "dir_path"; /**< unique key to access */ - using prop_tag = str_prop_tag; /**< property type */ + explicit DirPath(const std::string &path) : PathProperty(path) { set(path); } /** - * @brief check if given value is valid + * @brief Construct a new Dir Path object * - * @param v value to check - * @return bool true if valid + * @param path path to set */ - bool isValid(const std::string &v) const override; + explicit DirPath(const std::filesystem::path &path) : PathProperty(path) {} + static constexpr const char *key = "dir_path"; /**< unique key to access */ + using prop_tag = path_prop_tag; /**< property type */ /** - * @brief setter + * @brief check if given value is valid * - * @param v value to set + * @param v value to check + * @return bool true if valid */ - void set(const std::string &v) override; + bool isValid(const std::filesystem::path &v) const override; }; /** diff --git a/nntrainer/utils/base_properties.cpp b/nntrainer/utils/base_properties.cpp index ebd33a2317..0424a714d7 100644 --- a/nntrainer/utils/base_properties.cpp +++ b/nntrainer/utils/base_properties.cpp @@ -52,6 +52,39 @@ bool isFileReadAccessisble(fs::path p, std::error_code &ec) { return file.good(); #endif } + +/** + * @brief Helper for testing path kind looking behind symlinks. + */ +template +bool isPathKindHelper(const fs::path v, FileCheckFn_ &&file_check_fn) noexcept { + // Reject empty and non-existing paths + { + std::error_code ec; + if (v.empty() || !exists(v, ec)) + return false; + + if (ec) + return false; + } + + // Check if it is a path is of file_check_fn kind + { + std::error_code ec; + auto real_path = is_symlink(v) ? read_symlink(v, ec) : v; + + if (ec) + return false; + + if (!file_check_fn(real_path, ec)) + return false; + + if (ec) + return false; + } + + return true; +} } // namespace namespace nntrainer { @@ -197,32 +230,13 @@ TensorDim str_converter::from_string( PathProperty::~PathProperty() = default; bool PathProperty::isRegularFile(const fs::path &v) noexcept { - // Reject empty and non-existing paths - { - std::error_code ec; - if (v.empty() || !exists(v, ec)) - return false; - - if (ec) - return false; - } - - // Check if it is a path to regular file - { - std::error_code ec; - auto real_path = is_symlink(v)? read_symlink(v, ec) : v; - - if (ec) - return false; - - if (!is_regular_file(real_path, ec)) - return false; - - if (ec) - return false; - } + return isPathKindHelper( + v, [](const auto &v, auto ec) { return fs::is_regular_file(v, ec); }); +} - return true; +bool PathProperty::isDirectory(const fs::path &v) noexcept { + return isPathKindHelper( + v, [](const auto &v, auto ec) { return fs::is_directory(v, ec); }); } bool PathProperty::isReadAccessible(const fs::path &v) noexcept { diff --git a/nntrainer/utils/base_properties.h b/nntrainer/utils/base_properties.h index 45e15e9d26..92af279c5b 100644 --- a/nntrainer/utils/base_properties.h +++ b/nntrainer/utils/base_properties.h @@ -296,6 +296,7 @@ struct PathProperty : Property { protected: static bool isRegularFile(const std::filesystem::path &) noexcept; + static bool isDirectory(const std::filesystem::path &) noexcept; static bool isReadAccessible(const std::filesystem::path &) noexcept; }; From 9cd7922c319c998dd724ce1f529b409e0f79241f Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Wed, 12 Feb 2025 12:34:06 +0100 Subject: [PATCH 10/12] gtest.wrap Signed-off-by: Piotr Rak --- third_party/gtest.wrap | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 third_party/gtest.wrap diff --git a/third_party/gtest.wrap b/third_party/gtest.wrap new file mode 100644 index 0000000000..197210c6a6 --- /dev/null +++ b/third_party/gtest.wrap @@ -0,0 +1,16 @@ +[wrap-file] +directory = googletest-1.15.2 +source_url = https://github.com/google/googletest/archive/refs/tags/v1.15.2.tar.gz +source_filename = gtest-1.15.2.tar.gz +source_hash = 7b42b4d6ed48810c5362c265a17faebe90dc2373c885e5216439d37927f02926 +patch_filename = gtest_1.15.2-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/gtest_1.15.2-1/get_patch +patch_hash = b41cba05fc61d47b2ba5bf95732eb86ce2b67303f291b68a9587b7563c318141 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/gtest_1.15.2-1/gtest-1.15.2.tar.gz +wrapdb_version = 1.15.2-1 + +[provide] +gtest = gtest_dep +gtest_main = gtest_main_dep +gmock = gmock_dep +gmock_main = gmock_main_dep From 82ea1bd0ecee711c78ebb5e71d6700e4b589f52a Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Wed, 12 Feb 2025 14:40:57 +0100 Subject: [PATCH 11/12] [Prop] Switch model Save* properties to PathProperty Signed-off-by: Piotr Rak --- nntrainer/models/model_common_properties.h | 6 ++---- nntrainer/models/neuralnet.cpp | 13 ++++++------- nntrainer/utils/base_properties.h | 5 +++++ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/nntrainer/models/model_common_properties.h b/nntrainer/models/model_common_properties.h index 3bd8d9078b..fda1db0123 100644 --- a/nntrainer/models/model_common_properties.h +++ b/nntrainer/models/model_common_properties.h @@ -56,21 +56,19 @@ class LossType : public Property { * @brief model save path property * */ -class SavePath : public Property { +class SavePath : public PathProperty { public: static constexpr const char *key = "save_path"; /**< unique key to access */ - using prop_tag = str_prop_tag; /**< property type */ }; /** * @brief model save path property * */ -class SaveBestPath : public Property { +class SaveBestPath : public PathProperty { public: static constexpr const char *key = "save_best_path"; /**< unique key to access */ - using prop_tag = str_prop_tag; /**< property type */ }; /** diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp index 019d7ff9f1..de44aa0a07 100644 --- a/nntrainer/models/neuralnet.cpp +++ b/nntrainer/models/neuralnet.cpp @@ -624,14 +624,13 @@ void NeuralNetwork::save(const std::string &file_path, break; case ml::train::ModelFormat::MODEL_FORMAT_INI_WITH_BIN: { - auto old_save_path = std::get(model_flex_props); - auto bin_file_name = - file_path.substr(0, file_path.find_last_of('.')) + ".bin"; - - std::get(model_flex_props).set(bin_file_name); + auto old_save_path_prop = std::get(model_flex_props); + auto bin_file_path = old_save_path_prop.get(); + bin_file_path.replace_extension("bin"); + std::get(model_flex_props).set(bin_file_path); save(file_path, ml::train::ModelFormat::MODEL_FORMAT_INI); - save(bin_file_name, ml::train::ModelFormat::MODEL_FORMAT_BIN); - std::get(model_flex_props) = old_save_path; + save(bin_file_path, ml::train::ModelFormat::MODEL_FORMAT_BIN); + std::get(model_flex_props) = old_save_path_prop; break; } default: diff --git a/nntrainer/utils/base_properties.h b/nntrainer/utils/base_properties.h index 92af279c5b..81fc0b0c26 100644 --- a/nntrainer/utils/base_properties.h +++ b/nntrainer/utils/base_properties.h @@ -288,6 +288,11 @@ struct PathProperty : Property { explicit PathProperty(const std::string &str) : base(str) {} + /** + * @brief conversion operator to string + */ + operator typename std::filesystem::path::string_type() const { return get(); } + virtual bool isValid(const std::filesystem::path &value) const override { return true; }; From da3a5300fe4cf7bd9c1088152e288261695537ac Mon Sep 17 00:00:00 2001 From: Piotr Rak Date: Wed, 12 Feb 2025 15:06:02 +0100 Subject: [PATCH 12/12] clang-format --- nntrainer/models/model_common_properties.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nntrainer/models/model_common_properties.h b/nntrainer/models/model_common_properties.h index fda1db0123..572592df46 100644 --- a/nntrainer/models/model_common_properties.h +++ b/nntrainer/models/model_common_properties.h @@ -68,7 +68,7 @@ class SavePath : public PathProperty { class SaveBestPath : public PathProperty { public: static constexpr const char *key = - "save_best_path"; /**< unique key to access */ + "save_best_path"; /**< unique key to access */ }; /**