From a200308faee709339c9c6484c7ef7f6b3575e989 Mon Sep 17 00:00:00 2001 From: Seungbaek Hong Date: Tue, 21 Jan 2025 20:29:31 +0900 Subject: [PATCH] [ONNX] add basic onnx interpreter A simplest basic structure of onnx_interpreter has been added. In this implementation, only input layer, weight layer, add layer are supported. Basically, it is implemented by reading and parsing the onnx file using protobuf, and then creating an NNTrainer graph by sequentially checking the nodes. Users can simply specify the path to the ONNX model file through the "loadONNX" NNTrainer API function. The types of tensors handled during the process of creating a graph can be classified into input tensor, constant tensor, weight tensor. Among these, the current implementation only supports input tensor and weight tensor. In ONNX, weight tensors have initializers unlike other tensors, so it can be distinguished as weight tensors through initializers. In the current implementation, initializer list are managed separately as a vector, but this vector will be removed as the implementation is completed in the future. This commit uploads a minimal working implementation, and the onnx interpreter needs to be continuously updated. An example application of reading an onnx file to create an NNTrainer model using this interpreter, and a document setting up the execution environment by installing protobuf will be uploaded as separate commits. **Self evaluation:** Build test: [x]Passed [ ]Failed [ ]Skipped Run test: [x]Passed [ ]Failed [ ]Skipped Signed-off-by: Seungbaek Hong --- api/ccapi/include/onnx.h | 37 ++++ api/ccapi/meson.build | 1 + api/ccapi/src/factory.cpp | 11 ++ debian/ccapi-ml-training-dev.install | 1 + meson.build | 4 + nntrainer/schema/meson.build | 2 + nntrainer/schema/onnx_interpreter.h | 249 +++++++++++++++++++++++++++ packaging/nntrainer.spec | 2 + 8 files changed, 307 insertions(+) create mode 100644 api/ccapi/include/onnx.h create mode 100644 nntrainer/schema/onnx_interpreter.h diff --git a/api/ccapi/include/onnx.h b/api/ccapi/include/onnx.h new file mode 100644 index 0000000000..9740d05f40 --- /dev/null +++ b/api/ccapi/include/onnx.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2025 SeungBaek Hong + * + * @file onnx.h + * @date 12 February 2025 + * @see https://github.com/nnstreamer/nntrainer + * @author SeungBaek Hong + * @bug No known bugs except for NYI items + * @brief This is onnx converter interface for c++ API + */ + +#ifndef __ML_TRAIN_ONNX_H__ +#define __ML_TRAIN_ONNX_H__ + +#if __cplusplus >= MIN_CPP_VERSION + +#include + +namespace ml { +namespace train { + +/** + * @brief load model from onnx file + * + * @param path path of the onnx file to be loaded + * @return std::unique_ptr + */ +std::unique_ptr loadONNX(const std::string &path); + +} // namespace train +} // namespace ml + +#else +#error "CPP versions c++17 or over are only supported" +#endif // __cpluscplus +#endif // __ML_TRAIN_ONNX_H__ diff --git a/api/ccapi/meson.build b/api/ccapi/meson.build index 11bbd858dc..4d2801a4d7 100644 --- a/api/ccapi/meson.build +++ b/api/ccapi/meson.build @@ -17,6 +17,7 @@ ccapi_headers += meson.current_source_dir() / 'include' / 'model.h' ccapi_headers += meson.current_source_dir() / 'include' / 'optimizer.h' ccapi_headers += meson.current_source_dir() / 'include' / 'tensor_dim.h' ccapi_headers += meson.current_source_dir() / 'include' / 'tensor_api.h' +ccapi_headers += meson.current_source_dir() / 'include' / 'onnx.h' ccapi_headers += meson.current_source_dir() / '..' / 'nntrainer-api-common.h' ccapi_deps = [ diff --git a/api/ccapi/src/factory.cpp b/api/ccapi/src/factory.cpp index daad2c7b6f..946b797144 100644 --- a/api/ccapi/src/factory.cpp +++ b/api/ccapi/src/factory.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -141,6 +142,16 @@ createLearningRateScheduler(const std::string &type, return ac.createObject(type, properties); } +std::unique_ptr loadONNX(const std::string &path) { +#ifdef ENABLE_ONNX_INTERPRETER + nntrainer::ONNXInterpreter onnx = nntrainer::ONNXInterpreter(); + std::unique_ptr model = onnx.load(path); + return model; +#else + throw std::runtime_error("enable-onnx-interpreter is not enabled"); +#endif +} + std::string getVersion() { std::string version = std::to_string(VERSION_MAJOR); version += "."; diff --git a/debian/ccapi-ml-training-dev.install b/debian/ccapi-ml-training-dev.install index 2a315527ea..00a06446cf 100644 --- a/debian/ccapi-ml-training-dev.install +++ b/debian/ccapi-ml-training-dev.install @@ -5,5 +5,6 @@ /usr/include/nntrainer/dataset.h /usr/include/nntrainer/tensor_dim.h /usr/include/nntrainer/tensor_api.h +/usr/include/nntrainer/onnx.h /usr/lib/*/pkgconfig/ccapi-ml-training.pc /usr/lib/*/libccapi-*.a diff --git a/meson.build b/meson.build index acde363bbc..58ce719bb0 100644 --- a/meson.build +++ b/meson.build @@ -418,6 +418,10 @@ if get_option('enable-tflite-interpreter') extra_defines += '-DENABLE_TFLITE_INTERPRETER=1' endif +if get_option('enable-onnx-interpreter') + extra_defines += '-DENABLE_ONNX_INTERPRETER=1' +endif + opencv_dep = dummy_dep if get_option('platform') != 'android' diff --git a/nntrainer/schema/meson.build b/nntrainer/schema/meson.build index 8c037f27eb..294b82defc 100644 --- a/nntrainer/schema/meson.build +++ b/nntrainer/schema/meson.build @@ -69,3 +69,5 @@ endif foreach s : schema_sources nntrainer_sources += meson.current_source_dir() / s endforeach + +nntrainer_headers += meson.current_source_dir() / 'onnx_interpreter.h' \ No newline at end of file diff --git a/nntrainer/schema/onnx_interpreter.h b/nntrainer/schema/onnx_interpreter.h new file mode 100644 index 0000000000..272a3d9842 --- /dev/null +++ b/nntrainer/schema/onnx_interpreter.h @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2025 SeungBaek Hong + * + * @file onnx_interpreter.h + * @date 12 February 2025 + * @see https://github.com/nnstreamer/nntrainer + * @author SeungBaek Hong + * @bug No known bugs except for NYI items + * @brief This is onnx converter interface for c++ API + */ + +#ifndef __ONNX_INTERPRETER_H__ +#define __ONNX_INTERPRETER_H__ +#ifdef ENABLE_ONNX_INTERPRETER + +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @brief make "key=value" from key and value + * + * @tparam T type of a value + * @param key key + * @param value value + * @return std::string with "key=value" + */ +template +static std::string withKey(const std::string &key, const T &value) { + std::stringstream ss; + ss << key << "=" << value; + return ss.str(); +} + +/** + * @brief make "key=value1,value2,...valueN" from key and multiple values + * + * @tparam T type of a value + * @param key key + * @param value list of values + * @return std::string with "key=value1,value2,...valueN" + */ +template +static std::string withKey(const std::string &key, + std::initializer_list value) { + if (std::empty(value)) { + throw std::invalid_argument("empty data cannot be converted"); + } + std::stringstream ss; + ss << key << "="; + auto iter = value.begin(); + for (; iter != value.end() - 1; ++iter) { + ss << *iter << ','; + } + ss << *iter; + return ss.str(); +} + +namespace nntrainer { +/** + * @brief ONNX Interpreter class for converting onnx model to nntrainer model. + * + */ +class ONNXInterpreter { +public: + /** + * @brief Construct a new ONNXInterpreter object + * + */ + ONNXInterpreter(){}; + + /** + * @brief Destroy the ONNXInterpreter object + * + */ + ~ONNXInterpreter(){}; + + /** + * @brief Load onnx model from given path and convert to nntrainer model. + * + * @param path path of onnx model file. + */ + std::unique_ptr load(std::string path) { + // Load and parse onnx file with protobuf + std::ifstream file(path, std::ios::binary); + onnx_model.ParseFromIstream(&file); + + // Create nntrainer model instance + nntrainer_model = ml::train::createModel(); + std::vector> layers; + + // Create initializer(weight) unordered map and create weight layer + for (auto &initializer : onnx_model.graph().initializer()) { + // initializers are used to identify weights in the model + initializers.insert({cleanName(initializer.name()), initializer}); + std::string dim = transformDimString(initializer); + + // weight layer should be modified not to use input_shape as a parameter + layers.push_back(ml::train::createLayer( + "weight", {withKey("name", cleanName(initializer.name())), + withKey("dim", dim), withKey("input_shape", dim)})); + } + + // Create input & constant tensor layer + for (const auto &input : onnx_model.graph().input()) { + auto shape = input.type().tensor_type().shape(); + if (shape.dim_size() >= 4 || shape.dim_size() == 0) { + throw std::runtime_error( + "Tensors with batch dimensions of 5 or more, or zero_dimensional " + "tensors are not supported."); + } + + std::string dim = transformDimString(shape); + if (input.name().find("input") != + std::string::npos) { // Create input layer + layers.push_back(ml::train::createLayer( + "input", {withKey("name", cleanName(input.name())), + withKey("input_shape", dim)})); + } else { // Create constant tensor layer + throw std::runtime_error("Constant tensors are not supported yet."); + } + } + + // Create graph + for (const auto &node : onnx_model.graph().node()) { + /** + * @brief While NNTrainer represents graphs as connections between + * operations, ONNX represents graphs as connections between operations + * and tensors, requiring remapping of the names of output tensors from + * operations. + */ + std::vector inputNames; + auto outputRemap = [this](std::string &input_layer_name) { + if (layerOutputMap.find(input_layer_name) != layerOutputMap.end()) { + input_layer_name = layerOutputMap.find(input_layer_name)->second; + } + }; + for (auto &input : node.input()) { + std::string inputName = cleanName(input); + outputRemap(inputName); + inputNames.push_back(inputName); + } + + if (node.op_type() == "Add") { + layerOutputMap.insert( + {cleanName(node.output()[0]), cleanName(node.name())}); + + layers.push_back(ml::train::createLayer( + "add", + {"name=" + cleanName(node.name()), + withKey("input_layers", inputNames[0] + "," + inputNames[1])})); + } else { + throw std::runtime_error("Unsupported operation type: " + + node.op_type()); + } + } + + for (auto &layer : layers) { + nntrainer_model->addLayer(layer); + } + + return std::move(nntrainer_model); + }; + + /** + * @brief Clean the name of the layer to be used in nntrainer model + * + * @param name name of the layer + */ + std::string cleanName(std::string name) { + if (!name.empty() && name[0] == '/') { + name.erase(0, 1); + } + std::replace(name.begin(), name.end(), '/', '_'); + std::replace(name.begin(), name.end(), '.', '_'); + std::transform(name.begin(), name.end(), name.begin(), + [](unsigned char c) { return std::tolower(c); }); + return name; + } + + /** + * @brief Transform dimension string to nntrainer's format. + * + * @param shape ONNX TensorShapeProto + */ + std::string transformDimString(onnx::TensorShapeProto shape) { + std::string dim = ""; + for (int i = 0; i < shape.dim_size(); ++i) { + if (shape.dim()[i].has_dim_param()) { + throw std::runtime_error("Dynamic dimensions are not supported"); + } + dim += std::to_string(shape.dim()[i].dim_value()); + if (i < shape.dim_size() - 1) { + dim += ":"; + } + } + + if (shape.dim_size() == 1) { + dim = "1:1:" + dim; + } else if (shape.dim_size() == 2) { + dim = "1:" + dim; + } + + return dim; + } + + // /** + // * @brief Transform dimension string to nntrainer's format. + // * + // * @param initializer ONNX TensorProto + // */ + std::string transformDimString(onnx::TensorProto initializer) { + std::string dim = ""; + for (int i = 0; i < initializer.dims_size(); ++i) { + dim += std::to_string(initializer.dims()[i]); + if (i < initializer.dims_size() - 1) { + dim += ":"; + } + } + + if (initializer.dims_size() == 1) { + dim = "1:1:" + dim; + } else if (initializer.dims_size() == 2) { + dim = "1:" + dim; + } + + return dim; + }; + +private: + onnx::ModelProto onnx_model; // parsed onnx model + std::unique_ptr + nntrainer_model; // converted nntrainer model + std::unordered_map + layerOutputMap; // key: name of output, value: name of layer + std::unordered_map + initializers; // initializers are used to identify weights +}; + +} // namespace nntrainer + +#endif // ENABLE_ONNX_INTERPRETER +#endif // __ONNX_INTERPRETER_H__ diff --git a/packaging/nntrainer.spec b/packaging/nntrainer.spec index d6fb42f8a5..3766cd6147 100644 --- a/packaging/nntrainer.spec +++ b/packaging/nntrainer.spec @@ -538,6 +538,8 @@ cp -r result %{buildroot}%{_datadir}/nntrainer/unittest/ %{_includedir}/nntrainer/common_properties.h %{_includedir}/nntrainer/base_properties.h %{_includedir}/nntrainer/node_exporter.h +%{_includedir}/nntrainer/onnx.h +%{_includedir}/nntrainer/onnx_interpreter.h %{_includedir}/nntrainer/nntr_threads.h %{_includedir}/nntrainer/profiler.h # tensor headers