diff --git a/meson.build b/meson.build index acde363bb..9736effc2 100644 --- a/meson.build +++ b/meson.build @@ -21,11 +21,18 @@ add_project_arguments('-DVERSION_MAJOR=' + nntrainer_version_split[0], language: add_project_arguments('-DVERSION_MINOR=' + nntrainer_version_split[1], language: ['c', 'cpp']) add_project_arguments('-DVERSION_MICRO=' + nntrainer_version_split[2], language: ['c', 'cpp']) +if get_option('enable-test')# and get_option('enable-openvino-backbone').enabled() + gtest_proj = subproject('gtest') +endif + extra_defines = ['-DMIN_CPP_VERSION=201703L'] cc = meson.get_compiler('c') cxx = meson.get_compiler('cpp') +# Older libc++ has in seperate library +opt_stdcpp_fs_dep = cxx.find_library('stdc++fs', required: false) + if get_option('platform') == 'tizen' # Pass __TIZEN__ to the compiler add_project_arguments('-D__TIZEN__=1', language:['c','cpp']) @@ -493,7 +500,7 @@ if get_option('enable-app') endif endif -if get_option('platform') != 'android' +if get_option('platform') != 'android' and get_option('enable-nnstreamer-backbone') nnstreamer_dep = dependency('nnstreamer') message('building nnstreamer') subdir('nnstreamer') diff --git a/nntrainer/dataset/raw_file_data_producer.cpp b/nntrainer/dataset/raw_file_data_producer.cpp index e26b3f6fb..69ae7f262 100644 --- a/nntrainer/dataset/raw_file_data_producer.cpp +++ b/nntrainer/dataset/raw_file_data_producer.cpp @@ -109,8 +109,8 @@ RawFileDataProducer::size(const std::vector &input_dims, // << " Given file does not align with the given sample size, sample size: " // << sample_size << " file_size: " << file_size; - return static_cast(file_size) / - (sample_size * RawFileDataProducer::pixel_size); + return static_cast( + file_size / (sample_size * RawFileDataProducer::pixel_size)); } void RawFileDataProducer::exportTo( diff --git a/nntrainer/layers/common_properties.cpp b/nntrainer/layers/common_properties.cpp index 3ce4208d7..23a55eb75 100644 --- a/nntrainer/layers/common_properties.cpp +++ b/nntrainer/layers/common_properties.cpp @@ -20,10 +20,11 @@ #include #include -#include #include #include +namespace fs = std::filesystem; + namespace nntrainer { namespace props { @@ -56,30 +57,22 @@ void RandomTranslate::set(const float &value) { Property::set(std::abs(value)); } -bool FilePath::isValid(const std::string &v) const { - std::ifstream file(v, std::ios::binary | std::ios::ate); - return file.good(); -} - -void FilePath::set(const std::string &v) { - Property::set(v); - std::ifstream file(v, std::ios::binary | std::ios::ate); - cached_pos_size = file.tellg(); +bool FilePath::isValid(const fs::path &v) const { + const bool regular = PathProperty::isRegularFile(v); + const bool is_readable = PathProperty::isReadAccessible(v); + return regular && is_readable; } -std::ifstream::pos_type FilePath::file_size() { return cached_pos_size; } +std::uintmax_t FilePath::file_size() const { return fs::file_size(*this); } bool LossScaleForMixed::isValid(const float &value) const { return (value != 0); } -bool DirPath::isValid(const std::string &v) const { - struct stat dir; - return (stat(v.c_str(), &dir) == 0); +bool DirPath::isValid(const fs::path &v) const { + return PathProperty::isDirectory(v); } -void DirPath::set(const std::string &v) { Property::set(v); } - ReturnSequences::ReturnSequences(bool value) { set(value); } Bidirectional::Bidirectional(bool value) { set(value); } diff --git a/nntrainer/layers/common_properties.h b/nntrainer/layers/common_properties.h index 62440a584..22ad796ed 100644 --- a/nntrainer/layers/common_properties.h +++ b/nntrainer/layers/common_properties.h @@ -664,82 +664,79 @@ class RandomTranslate : public nntrainer::Property { * @brief Props containing file path value * */ -class FilePath : public Property { +class FilePath : public PathProperty { public: /** * @brief Construct a new File Path object */ - FilePath() : Property() {} + FilePath() = default; /** * @brief Construct a new File Path object * * @param path path to set */ - FilePath(const std::string &path) { set(path); } - static constexpr const char *key = "path"; /**< unique key to access */ - using prop_tag = str_prop_tag; /**< property type */ + explicit FilePath(const std::string &path) : PathProperty(path) {} /** - * @brief check if given value is valid + * @brief Construct a new File Path object * - * @param v value to check - * @return bool true if valid + * @param path path to set */ - bool isValid(const std::string &v) const override; + explicit FilePath(const std::filesystem::path &path) : PathProperty(path) {} + static constexpr const char *key = "path"; /**< unique key to access */ + using prop_tag = path_prop_tag; /**< property type */ /** - * @brief setter + * @brief check if given value is valid * - * @param v value to set + * @param v value to check + * @return bool true if valid */ - void set(const std::string &v) override; + bool isValid(const std::filesystem::path &v) const override; /** * @brief return file size * - * @return std::ifstream::pos_type size of the file + * @return std::uintmax_t size of the file */ - std::ifstream::pos_type file_size(); - -private: - std::ifstream::pos_type cached_pos_size; + std::uintmax_t file_size() const; }; /** * @brief Props containing directory path value * */ -class DirPath : public Property { +class DirPath : public PathProperty { public: /** * @brief Construct a new Dir Path object */ - DirPath() : Property() {} + DirPath() = default; /** * @brief Construct a new Dir Path object * * @param path path to set */ - DirPath(const std::string &path) { set(path); } - static constexpr const char *key = "dir_path"; /**< unique key to access */ - using prop_tag = str_prop_tag; /**< property type */ + explicit DirPath(const std::string &path) : PathProperty(path) { set(path); } /** - * @brief check if given value is valid + * @brief Construct a new Dir Path object * - * @param v value to check - * @return bool true if valid + * @param path path to set */ - bool isValid(const std::string &v) const override; + explicit DirPath(const std::filesystem::path &path) : PathProperty(path) {} + static constexpr const char *key = "dir_path"; /**< unique key to access */ + using prop_tag = path_prop_tag; /**< property type */ /** - * @brief setter + * @brief check if given value is valid * - * @param v value to set + * @param v value to check + * @return bool true if valid */ - void set(const std::string &v) override; + bool isValid(const std::filesystem::path &v) const override; }; /** diff --git a/nntrainer/meson.build b/nntrainer/meson.build index 68f102101..4a4d2cc2b 100644 --- a/nntrainer/meson.build +++ b/nntrainer/meson.build @@ -27,7 +27,8 @@ nntrainer_base_deps=[ libm_dep, libdl_dep, thread_dep, - openmp_dep + openmp_dep, + opt_stdcpp_fs_dep ] if get_option('platform') == 'tizen' diff --git a/nntrainer/models/model_common_properties.h b/nntrainer/models/model_common_properties.h index 3bd8d9078..572592df4 100644 --- a/nntrainer/models/model_common_properties.h +++ b/nntrainer/models/model_common_properties.h @@ -56,21 +56,19 @@ class LossType : public Property { * @brief model save path property * */ -class SavePath : public Property { +class SavePath : public PathProperty { public: static constexpr const char *key = "save_path"; /**< unique key to access */ - using prop_tag = str_prop_tag; /**< property type */ }; /** * @brief model save path property * */ -class SaveBestPath : public Property { +class SaveBestPath : public PathProperty { public: static constexpr const char *key = - "save_best_path"; /**< unique key to access */ - using prop_tag = str_prop_tag; /**< property type */ + "save_best_path"; /**< unique key to access */ }; /** diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp index 019d7ff9f..de44aa0a0 100644 --- a/nntrainer/models/neuralnet.cpp +++ b/nntrainer/models/neuralnet.cpp @@ -624,14 +624,13 @@ void NeuralNetwork::save(const std::string &file_path, break; case ml::train::ModelFormat::MODEL_FORMAT_INI_WITH_BIN: { - auto old_save_path = std::get(model_flex_props); - auto bin_file_name = - file_path.substr(0, file_path.find_last_of('.')) + ".bin"; - - std::get(model_flex_props).set(bin_file_name); + auto old_save_path_prop = std::get(model_flex_props); + auto bin_file_path = old_save_path_prop.get(); + bin_file_path.replace_extension("bin"); + std::get(model_flex_props).set(bin_file_path); save(file_path, ml::train::ModelFormat::MODEL_FORMAT_INI); - save(bin_file_name, ml::train::ModelFormat::MODEL_FORMAT_BIN); - std::get(model_flex_props) = old_save_path; + save(bin_file_path, ml::train::ModelFormat::MODEL_FORMAT_BIN); + std::get(model_flex_props) = old_save_path_prop; break; } default: diff --git a/nntrainer/utils/base_properties.cpp b/nntrainer/utils/base_properties.cpp index 30e0cc334..0424a714d 100644 --- a/nntrainer/utils/base_properties.cpp +++ b/nntrainer/utils/base_properties.cpp @@ -11,11 +11,82 @@ */ #include +#include #include #include #include +#include #include +#if !defined(_WIN32) +#include +#endif + +namespace fs = std::filesystem; + +namespace { + +/** + * @brief constructs system error_code from POSIX errno + */ +[[maybe_unused]] auto make_system_error_code() { + return std::error_code{errno, std::system_category()}; +} + +bool isFileReadAccessisble(fs::path p, std::error_code &ec) { +#if defined(_POSIX_VERSION) + // Check if we have read permissions to path pointing this file + auto r = ::access(p.c_str(), R_OK); + if (r == 0) { + ec = std::error_code{}; + return true; + } + + ec = make_system_error_code(); + + return false; +#else + ec = std::error_code{}; + // Unless it is POSIX, best bet is to try it + std::ifstream file(p, std::ios::binary | std::ios::ate); + return file.good(); +#endif +} + +/** + * @brief Helper for testing path kind looking behind symlinks. + */ +template +bool isPathKindHelper(const fs::path v, FileCheckFn_ &&file_check_fn) noexcept { + // Reject empty and non-existing paths + { + std::error_code ec; + if (v.empty() || !exists(v, ec)) + return false; + + if (ec) + return false; + } + + // Check if it is a path is of file_check_fn kind + { + std::error_code ec; + auto real_path = is_symlink(v) ? read_symlink(v, ec) : v; + + if (ec) + return false; + + if (!file_check_fn(real_path, ec)) + return false; + + if (ec) + return false; + } + + return true; +} +} // namespace + namespace nntrainer { bool PositiveIntegerProperty::isValid(const unsigned int &value) const { return value > 0; @@ -33,6 +104,18 @@ std::string str_converter::from_string( return value; } +template <> +std::string +str_converter::to_string(const fs::path &value) { + return value; +} + +template <> +fs::path +str_converter::from_string(const std::string &value) { + return fs::path{value}; +} + template <> std::string str_converter::to_string(const bool &value) { return value ? "true" : "false"; @@ -144,4 +227,25 @@ TensorDim str_converter::from_string( return target; } +PathProperty::~PathProperty() = default; + +bool PathProperty::isRegularFile(const fs::path &v) noexcept { + return isPathKindHelper( + v, [](const auto &v, auto ec) { return fs::is_regular_file(v, ec); }); +} + +bool PathProperty::isDirectory(const fs::path &v) noexcept { + return isPathKindHelper( + v, [](const auto &v, auto ec) { return fs::is_directory(v, ec); }); +} + +bool PathProperty::isReadAccessible(const fs::path &v) noexcept { + std::error_code ec; + + if (!isFileReadAccessisble(v, ec)) + return false; + + return !ec; +} + } // namespace nntrainer diff --git a/nntrainer/utils/base_properties.h b/nntrainer/utils/base_properties.h index ceee6bef8..81fc0b0c2 100644 --- a/nntrainer/utils/base_properties.h +++ b/nntrainer/utils/base_properties.h @@ -12,11 +12,15 @@ #ifndef __BASE_PROPERTIES_H__ #define __BASE_PROPERTIES_H__ +#include #include +#include #include #include +#include #include #include +#include #include #include @@ -116,6 +120,12 @@ struct double_prop_tag {}; */ struct str_prop_tag {}; +/** + * @brief property is treated as a path + * + */ +struct path_prop_tag {}; + /** * @brief property is treated as boolean * @@ -266,6 +276,35 @@ template class Property { std::unique_ptr value; /**< underlying data */ }; +/** + * @brief The filesystem path property + */ +struct PathProperty : Property { + using base = Property; + + using base::base; + + using prop_tag = path_prop_tag; + + explicit PathProperty(const std::string &str) : base(str) {} + + /** + * @brief conversion operator to string + */ + operator typename std::filesystem::path::string_type() const { return get(); } + + virtual bool isValid(const std::filesystem::path &value) const override { + return true; + }; + + virtual ~PathProperty() override; + +protected: + static bool isRegularFile(const std::filesystem::path &) noexcept; + static bool isDirectory(const std::filesystem::path &) noexcept; + static bool isReadAccessible(const std::filesystem::path &) noexcept; +}; + /** * @brief enum property * @@ -461,6 +500,21 @@ template <> std::string str_converter::from_string(const std::string &value); +/** + * @copydoc template struct str_converter + */ +template <> +std::string str_converter::to_string( + const std::filesystem::path &value); + +/** + * @copydoc template struct str_converter + */ +template <> +std::filesystem::path +str_converter::from_string( + const std::string &value); + /** * @copydoc template struct str_converter */ @@ -720,6 +774,125 @@ class TensorFormat final : public EnumProperty { }; }; +/** + * @class ICaseLexOrder + * + * @brief The Lexicograpthic order strings disregarding case of the characters. + */ +struct ICaseLexOrder final { + /** + * @brief case insensitive character compare + * + * @return true if @param l is less than @param l + */ + constexpr static bool compare(unsigned char l, unsigned char r) noexcept { + return std::less{}(std::tolower(l), std::tolower(r)); + } + + /** + * @brief case insensitive character compare + * + * @return true if @param l is less than @param l + */ + constexpr bool operator()(unsigned char l, unsigned char r) const noexcept { + return ICaseLexOrder::compare(l, r); + } + + /** + * @brief case insensitive character compare + * + * @return true if @param l is less than @param l + */ + static bool compare(std::string_view l, std::string_view r) noexcept { + return std::lexicographical_compare(l.begin(), l.end(), r.begin(), r.end(), + ICaseLexOrder{}); + } + + /** + * @brief case insensitive character compare + * + * @return true if @param l is less than @param l + */ + bool operator()(std::string_view l, std::string_view r) const noexcept { + return ICaseLexOrder::compare(l, r); + } +}; + +/** + * @brief case-insensitive set of string views + */ +using ICaseStringSet = std::set; + +/** + * @brief Declares allowed model format extensions + * + * To specify allowed file extensions it is required to specialize: + * + * @code{.cpp} + * struct my_model_tag {}; + * + * template <> AllowedModelFormatExtSet::extensions = { + * "file_ext1"sv, "file_ext2"sv, // etc... + * }; + * + * // Those asserts should hold, given files: + * // - 'file.fILe_eXT1' + * // - '/bananas/file.FILE_ext2' + * // exist in the file-system and read-accessible regular files or symlinks + * // to thereof. + * { + * assert(ModelPathProperty{}.isValid("file.fILe_eXT1")); + * assert(ModelPathProperty{}.isValid("/bananas/file.FILE_ext2")); + * } + * + * @endcode + */ +template class AllowedModelFormatExtSet { + + static const ICaseStringSet extensions; + +public: + /** + * @brief check if the file extension (case-insensetive) is in the set + */ + constexpr static bool has(std::string_view ext) noexcept { + return extensions.find(ext) != extensions.end(); // c++20 std::set::contains + } +}; + +/** + * @brief Trait returing if model file extension is is allowed/supported. + */ +template struct model_extension_traits { + constexpr static bool isFileExtAllowed(std::string_view file_ext) noexcept { + return AllowedModelFormatExtSet::has(file_ext); + } +}; + +template class ModelPathProperty : public PathProperty { + +public: + using prop_tag = path_prop_tag; + + bool isValid(const std::filesystem::path &v) const override { + + if (!PathProperty::isRegularFile(v)) + return false; + + // Checked if file has allowed supported file extension + std::string ext = v.extension(); + + if (!model_extension_traits::isFileExtAllowed(ext)) + return false; + + // Reject path we don't have read access permission + if (!PathProperty::isReadAccessible(v)) + return false; + + return true; + } +}; + // /** // * @brief trainable property, use this to set and check how if certain layer // is diff --git a/nntrainer/utils/util_func.h b/nntrainer/utils/util_func.h index c041cabed..8d26790ff 100644 --- a/nntrainer/utils/util_func.h +++ b/nntrainer/utils/util_func.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include diff --git a/test/meson.build b/test/meson.build index b3d99e92c..4cc360bce 100644 --- a/test/meson.build +++ b/test/meson.build @@ -15,6 +15,7 @@ nntrainer_testutil_dep = declare_dependency( link_with: nntrainer_testutil_lib, include_directories: nntrainer_test_inc ) +gtest_subproj_main_dep = dependency('gtest_main', required: false) nntrainer_test_deps = [ gmock_dep, @@ -27,6 +28,7 @@ nntrainer_test_deps = [ nntrainer_test_main_deps = [ gmock_dep, gtest_main_dep, + gtest_subproj_main_dep, nntrainer_dep, nntrainer_testutil_dep ] diff --git a/test/unittest/compiler/meson.build b/test/unittest/compiler/meson.build index cf9a86457..6e028fe15 100644 --- a/test/unittest/compiler/meson.build +++ b/test/unittest/compiler/meson.build @@ -4,10 +4,13 @@ test_target = [ 'compiler_test_util.cpp', 'unittest_compiler.cpp', 'unittest_interpreter.cpp', - 'unittest_tflite_export.cpp', 'unittest_realizer.cpp', ] +if get_option('enable-tflite-interpreter') + test_target += 'unittest_tflite_export.cpp' +endif + exe = executable( test_name, test_target, diff --git a/test/unittest/integration_tests/meson.build b/test/unittest/integration_tests/meson.build index 8071a860f..50862610b 100644 --- a/test/unittest/integration_tests/meson.build +++ b/test/unittest/integration_tests/meson.build @@ -3,9 +3,12 @@ mixed_precision_test_name = 'integration_test_mixed_precision' test_target = [ 'integration_tests.cpp', - 'integration_test_loss.cpp', ] +if (get_option('enable-tflite-interpreter')) + test_target += 'integration_test_loss.cpp' +endif + mixed_precision_targets = [ model_util_path / 'models_test_utils.cpp', 'integration_test_mixed_precision.cpp', @@ -26,7 +29,7 @@ exe = executable( test_target, include_directories: [include_directories('.'), model_util_include_dir], dependencies: [ - nntrainer_test_main_deps, + nntrainer_test_deps, nntrainer_dep, nntrainer_ccapi_dep, ], diff --git a/test/unittest/unittest_base_properties.cpp b/test/unittest/unittest_base_properties.cpp index e25c97386..7c1e25daa 100644 --- a/test/unittest/unittest_base_properties.cpp +++ b/test/unittest/unittest_base_properties.cpp @@ -24,6 +24,8 @@ namespace { /**< define a property for testing */ +namespace fs = std::filesystem; + /** * @brief banana property tag for example * @@ -60,6 +62,22 @@ class QualityOfBanana : public nntrainer::Property { } }; +struct PathToNowhere : public nntrainer::PathProperty { + PathToNowhere() = default; + PathToNowhere(const fs::path &path); + static constexpr const char *key = "path_to_nowhere"; + + bool isValid(const fs::path &v) const override { return !exists(v); } +}; + +struct PathToBananaFarm : public nntrainer::PathProperty { + PathToBananaFarm() = default; + PathToBananaFarm(const fs::path &path); + static constexpr const char *key = "path_to_farm"; + + bool isValid(const fs::path &v) const override { return exists(v); } +}; + /** * @brief Property example to be used as a bool * @@ -271,6 +289,24 @@ TEST(BasicProperty, valid_p) { EXPECT_FLOAT_EQ(q.get(), 1.3245f); } + { /** set -> get / from_string, fs::path (doesn't exist) */ + const auto pth_banana = fs::path("/golden-BANANA-farm"); + + EXPECT_FALSE(exists(pth_banana)); + PathToNowhere p; + nntrainer::from_string("/golden-BANANA-farm", p); + EXPECT_EQ(pth_banana, p.get()); + } + + { /** set -> get / from_string, fs::path (path exists) */ + const auto cwd = + fs::current_path(); // We can be certain that process has cwd + EXPECT_TRUE(exists(cwd)); + PathToBananaFarm p; + p.set(cwd); + EXPECT_EQ(nntrainer::to_string(p), fs::current_path()); + } + { /**< enum type test from_string -> get */ BananaType t; nntrainer::from_string("CAVENDISH", t); diff --git a/third_party/gtest.wrap b/third_party/gtest.wrap new file mode 100644 index 000000000..197210c6a --- /dev/null +++ b/third_party/gtest.wrap @@ -0,0 +1,16 @@ +[wrap-file] +directory = googletest-1.15.2 +source_url = https://github.com/google/googletest/archive/refs/tags/v1.15.2.tar.gz +source_filename = gtest-1.15.2.tar.gz +source_hash = 7b42b4d6ed48810c5362c265a17faebe90dc2373c885e5216439d37927f02926 +patch_filename = gtest_1.15.2-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/gtest_1.15.2-1/get_patch +patch_hash = b41cba05fc61d47b2ba5bf95732eb86ce2b67303f291b68a9587b7563c318141 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/gtest_1.15.2-1/gtest-1.15.2.tar.gz +wrapdb_version = 1.15.2-1 + +[provide] +gtest = gtest_dep +gtest_main = gtest_main_dep +gmock = gmock_dep +gmock_main = gmock_main_dep