diff --git a/compiler/circle-interpreter-cffi-test/CMakeLists.txt b/compiler/circle-interpreter-cffi-test/CMakeLists.txt new file mode 100644 index 00000000000..f2704baaeb5 --- /dev/null +++ b/compiler/circle-interpreter-cffi-test/CMakeLists.txt @@ -0,0 +1,17 @@ +if(NOT ENABLE_TEST) + return() +endif(NOT ENABLE_TEST) + +set(VIRTUALENV "${NNCC_OVERLAY_DIR}/venv_2_12_1") +set(TEST_LIST_FILE "test.lst") + +get_target_property(ARTIFACTS_PATH testDataGenerator BINARY_DIR) + +add_test( + NAME circle_interpreter_cffi_test + COMMAND ${VIRTUALENV}/bin/python infer.py + --lib_path $ + --test_list ${TEST_LIST_FILE} + --artifact_dir ${ARTIFACTS_PATH} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) diff --git a/compiler/circle-interpreter-cffi-test/README.md b/compiler/circle-interpreter-cffi-test/README.md new file mode 100644 index 00000000000..ec6f921d1be --- /dev/null +++ b/compiler/circle-interpreter-cffi-test/README.md @@ -0,0 +1,11 @@ +# circle-interpreter-cffi-test + +The `circle_interpereter_cffi` library wrapped with CFFI is designed to expose an existing `luci-interpreter` class to Python. It simplifies the integration of the class by creating a Python-compatible interface using CFFI. + +`circle-interpreter-cffi-test` ensures that the Python bindings for the C++ library correctly. Specifically, it verifies that: + +1. The CFFI-wrapped library can succesfully load the circle model. +2. Inputs passed from Python are correctly interpreted by the `luci-interpreter`. +3. The output generated by the interpreter matches the expected results. + +This test provides confidence that `luci-interpter`, when accessed through the Python interface, produces the same results as the original implementation. diff --git a/compiler/circle-interpreter-cffi-test/infer.py b/compiler/circle-interpreter-cffi-test/infer.py new file mode 100644 index 00000000000..b4e2aee8bc2 --- /dev/null +++ b/compiler/circle-interpreter-cffi-test/infer.py @@ -0,0 +1,112 @@ +import argparse +import h5py +import numpy as np +from pathlib import Path +import re +import sys + +############ Managing paths for the artifacts required by the test. + + +def extract_test_args(s): + p = re.compile('eval\\((.*)\\)') + result = p.search(s) + return result.group(1) + + +parser = argparse.ArgumentParser() +parser.add_argument('--lib_path', type=str, required=True) +parser.add_argument('--test_list', type=str, required=True) +parser.add_argument('--artifact_dir', type=str, required=True) +args = parser.parse_args() + +with open(args.test_list) as f: + contents = [line.rstrip() for line in f] +# remove newline and comments. +eval_lines = [line for line in contents if line.startswith('eval(')] +test_args = [extract_test_args(line) for line in eval_lines] +test_models = [Path(args.artifact_dir) / f'{arg}.circle' for arg in test_args] +input_data = [ + Path(args.artifact_dir) / f'{arg}.opt/metadata/tc/input.h5' for arg in test_args +] +expected_output_data = [ + Path(args.artifact_dir) / f'{arg}.opt/metadata/tc/expected.h5' for arg in test_args +] + +############ CFFI test + +from cffi import FFI + +ffi = FFI() +ffi.cdef(""" + typedef struct InterpreterWrapper InterpreterWrapper; + + const char *get_last_error(void); + void clear_last_error(void); + InterpreterWrapper *Interpreter_new(const uint8_t *data, const size_t data_size); + void Interpreter_delete(InterpreterWrapper *intp); + void Interpreter_interpret(InterpreterWrapper *intp); + void Interpreter_writeInputTensor(InterpreterWrapper *intp, const int input_idx, const void *data, size_t input_size); + void Interpreter_readOutputTensor(InterpreterWrapper *intp, const int output_idx, void *output, size_t output_size); +""") +C = ffi.dlopen(args.lib_path) + + +def check_for_errors(): + error_message = ffi.string(C.get_last_error()).decode('utf-8') + if error_message: + C.clear_last_error() + raise RuntimeError(f'C++ Exception: {error_message}') + + +def error_checked(func): + """ + Decorator to wrap functions with error checking. + """ + def wrapper(*args, **kwargs): + result = func(*args, **kwargs) + check_for_errors() + return result + + return wrapper + + +Interpreter_new = error_checked(C.Interpreter_new) +Interpreter_delete = error_checked(C.Interpreter_delete) +Interpreter_interpret = error_checked(C.Interpreter_interpret) +Interpreter_writeInputTensor = error_checked(C.Interpreter_writeInputTensor) +Interpreter_readOutputTensor = error_checked(C.Interpreter_readOutputTensor) + +for idx, model_path in enumerate(test_models): + with open(model_path, "rb") as f: + model_data = ffi.from_buffer(bytearray(f.read())) + + try: + intp = Interpreter_new(model_data, len(model_data)) + + # Set inputs + h5 = h5py.File(input_data[idx]) + input_values = h5.get('value') + input_num = len(input_values) + for input_idx in range(input_num): + arr = np.array(input_values.get(str(input_idx))) + c_arr = ffi.from_buffer(arr) + Interpreter_writeInputTensor(intp, input_idx, c_arr, arr.nbytes) + # Do inference + Interpreter_interpret(intp) + # Check outputs + h5 = h5py.File(expected_output_data[idx]) + output_values = h5.get('value') + output_num = len(output_values) + for output_idx in range(output_num): + arr = np.array(output_values.get(str(output_idx))) + result = np.empty(arr.shape, dtype=arr.dtype) + Interpreter_readOutputTensor(intp, output_idx, ffi.from_buffer(result), + arr.nbytes) + if not np.allclose(result, arr): + raise RuntimeError("Wrong outputs") + + Interpreter_delete(intp) + except RuntimeError as e: + print(e) + sys.exit(-1) diff --git a/compiler/circle-interpreter-cffi-test/requires.cmake b/compiler/circle-interpreter-cffi-test/requires.cmake new file mode 100644 index 00000000000..8d8585b470d --- /dev/null +++ b/compiler/circle-interpreter-cffi-test/requires.cmake @@ -0,0 +1,2 @@ +require("common-artifacts") +require("circle-interpreter") diff --git a/compiler/circle-interpreter-cffi-test/test.lst b/compiler/circle-interpreter-cffi-test/test.lst new file mode 100644 index 00000000000..97ec610ad5f --- /dev/null +++ b/compiler/circle-interpreter-cffi-test/test.lst @@ -0,0 +1,17 @@ +eval(Add_000) +eval(Add_U8_000) +eval(AveragePool2D_000) +eval(Concatenation_000) +eval(Conv2D_000) +eval(Conv2D_001) +eval(Conv2D_002) +eval(DepthwiseConv2D_000) +eval(FullyConnected_000) +eval(FullyConnected_001) +eval(MaxPool2D_000) +eval(Mul_000) +eval(Pad_000) +eval(Reshape_000) +eval(Reshape_001) +eval(Reshape_002) +eval(Softmax_000) diff --git a/compiler/circle-interpreter/CMakeLists.txt b/compiler/circle-interpreter/CMakeLists.txt index d18db3e11ea..768cdd8e357 100644 --- a/compiler/circle-interpreter/CMakeLists.txt +++ b/compiler/circle-interpreter/CMakeLists.txt @@ -11,3 +11,13 @@ target_link_libraries(circle-interpreter PRIVATE safemain) target_link_libraries(circle-interpreter PRIVATE vconone) install(TARGETS circle-interpreter DESTINATION bin) + +set(INTERPRETER_CFFI + src/CircleInterpreter_cffi.cpp +) + +add_library(circle_interpreter_cffi SHARED ${INTERPRETER_CFFI}) +target_link_libraries(circle_interpreter_cffi PRIVATE luci_import) +target_link_libraries(circle_interpreter_cffi PRIVATE luci_interpreter) + +install(TARGETS circle_interpreter_cffi DESTINATION lib) diff --git a/compiler/circle-interpreter/src/CircleInterpreter_cffi.cpp b/compiler/circle-interpreter/src/CircleInterpreter_cffi.cpp new file mode 100644 index 00000000000..e47a6375958 --- /dev/null +++ b/compiler/circle-interpreter/src/CircleInterpreter_cffi.cpp @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This library is for using luci-interpreter in Python. It is implemented + * with CFFI. CFFI is a FFI (Foreign Function Interface) for Python calling + * C code. + */ + +#include +#include + +#include +#include + +namespace +{ + +// Global variable for error message +std::string last_error_message; + +template size_t getTensorSize(const NodeT *node) +{ + uint32_t tensor_size = luci::size(node->dtype()); + for (uint32_t i = 0; i < node->rank(); ++i) + tensor_size *= node->dim(i).value(); + return tensor_size; +} + +// Function to retrieve the last error message. +extern "C" const char *get_last_error(void) { return last_error_message.c_str(); } + +// Clear the last error message +extern "C" void clear_last_error(void) { last_error_message.clear(); } + +/** + * @brief A function that wraps another function and catches any exceptions. + * + * This function executes the given callable (`func`) with the provided arguments. + * If the callable throws an exception, the exception message is stored in a + * `last_error_message` global variable. + * + * @tparam Func The type of the callable funciton. + * @tparam Args The types of arguments to pass to the callable function. + * @param func The callable function to execute. + * @param args The arguments to pass to the callable function. + * @return The return value of the callable function, or a default value in case of + * an exception. If the function has a `void` return type, it simply returns + * without any value. + * + * @note This function ensures that exceptions are safely caught and conveted to + * error messages that can be queried externally, e.g. from Python. + */ +template +auto exception_wrapper(Func func, Args... args) -> typename std::result_of::type +{ + using ReturnType = typename std::result_of::type; + + try + { + return func(std::forward(args)...); + } + catch (const std::exception &e) + { + last_error_message = e.what(); + if constexpr (not std::is_void::value) + { + return ReturnType{}; + } + } + catch (...) + { + last_error_message = "Unknown error"; + if constexpr (not std::is_void::value) + { + return ReturnType{}; + } + } +} + +} // namespace + +/* + * Q) Why do we need this wrapper class? + * + * A) This class is designed to address specific constraints introduced by + * the use of CFFI for Python bindings. The original class lacked the + * ability to maintain internal state like luci::Module because it is defined + * out of the class. But, in Python, there's no way to keep luci::Module in memory. + * To overcome this limitation, a wrapper class is implemented to manage and + * keep states efficiently. + * + * Moreover, the original interface relied on passing C++ classes as arguments, + * which posed compatibility challenges when exposed to Python via CFFI. To + * simplify the integration process, the wrapper class redesigns the interface + * to accept more generic inputs, such as primitive types or standard containers, + * ensuring seamless interaction between C++ and Python. + * + * Overall, the redesigned class or the wrapper class preserves the original + * functionality while introducing state management and a more flexible inteface, + * making it highly suitable for Python-C++ interoperability through CFFI. + */ +class InterpreterWrapper +{ +public: + explicit InterpreterWrapper(const uint8_t *data, const size_t data_size) + { + luci::Importer importer; + _module = importer.importModule(data, data_size); + if (_module == nullptr) + { + throw std::runtime_error{"Cannot import module."}; + } + _intp = new luci_interpreter::Interpreter(_module.get()); + } + + ~InterpreterWrapper() { delete _intp; } + + void interpret(void) { _intp->interpret(); } + + void writeInputTensor(const int input_idx, const void *data, size_t input_size) + { + const auto input_nodes = loco::input_nodes(_module->graph()); + const auto input_node = loco::must_cast(input_nodes.at(input_idx)); + // Input size from model binary + const auto fb_input_size = ::getTensorSize(input_node); + if (fb_input_size != input_size) + { + const auto msg = "Invalid input size: " + std::to_string(fb_input_size) + + " != " + std::to_string(input_size); + throw std::runtime_error(msg); + } + _intp->writeInputTensor(input_node, data, fb_input_size); + } + + void readOutputTensor(const int output_idx, void *output, size_t output_size) + { + const auto output_nodes = loco::output_nodes(_module->graph()); + const auto output_node = + loco::must_cast(output_nodes.at(output_idx)); + const auto fb_output_size = ::getTensorSize(output_node); + if (fb_output_size != output_size) + { + const auto msg = "Invalid output size: " + std::to_string(fb_output_size) + + " != " + std::to_string(output_size); + throw std::runtime_error(msg); + } + _intp->readOutputTensor(output_node, output, fb_output_size); + } + +private: + luci_interpreter::Interpreter *_intp = nullptr; + std::unique_ptr _module; +}; + +/* + * CFFI primarily uses functions instead of classes because it is designed to + * work with C-compatible interfaces. + * + * - This extern "C" is necessary to avoid name mangling. + * - Explicitly pass the object pointer to any funcitons that operates on the object. + */ +extern "C" { + +InterpreterWrapper *Interpreter_new(const uint8_t *data, const size_t data_size) +{ + return ::exception_wrapper([&]() { return new InterpreterWrapper(data, data_size); }); +} + +void Interpreter_delete(InterpreterWrapper *intp) +{ + ::exception_wrapper([&]() { delete intp; }); +} + +void Interpreter_interpret(InterpreterWrapper *intp) +{ + ::exception_wrapper([&]() { intp->interpret(); }); +} + +void Interpreter_writeInputTensor(InterpreterWrapper *intp, const int input_idx, const void *data, + size_t input_size) +{ + ::exception_wrapper([&]() { intp->writeInputTensor(input_idx, data, input_size); }); +} + +void Interpreter_readOutputTensor(InterpreterWrapper *intp, const int output_idx, void *output, + size_t output_size) +{ + ::exception_wrapper([&]() { intp->readOutputTensor(output_idx, output, output_size); }); +} + +} // extern "C" diff --git a/compiler/common-artifacts/CMakeLists.txt b/compiler/common-artifacts/CMakeLists.txt index 501146b351d..15dd6eb1877 100644 --- a/compiler/common-artifacts/CMakeLists.txt +++ b/compiler/common-artifacts/CMakeLists.txt @@ -76,6 +76,7 @@ if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} COMMAND ${CMAKE_COMMAND} -E echo "pytest==7.4.3" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} COMMAND ${CMAKE_COMMAND} -E echo "h5py==3.11.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} + COMMAND ${CMAKE_COMMAND} -E echo "cffi==1.16.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000 ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000 @@ -92,6 +93,7 @@ else(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64") COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} COMMAND ${CMAKE_COMMAND} -E echo "pytest==7.4.3" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} COMMAND ${CMAKE_COMMAND} -E echo "h5py==3.11.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} + COMMAND ${CMAKE_COMMAND} -E echo "cffi==1.16.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1} COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000 ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000