[DRAFT] Introduce circle-interpreter-cffi-test

compiler/circle-interpreter-cffi-test/CMakeLists.txt

@@ -0,0 +1,17 @@
+if(NOT ENABLE_TEST)
+  return()
+endif(NOT ENABLE_TEST)
+
+set(VIRTUALENV "${NNCC_OVERLAY_DIR}/venv_2_12_1")
+set(TEST_LIST_FILE "test.lst")
+
+get_target_property(ARTIFACTS_PATH testDataGenerator BINARY_DIR)
+
+add_test(
+  NAME circle_interpreter_cffi_test
+  COMMAND ${VIRTUALENV}/bin/python infer.py
+                    --lib_path $<TARGET_FILE:circle_interpreter_cffi>
+                    --test_list ${TEST_LIST_FILE}
+                    --artifact_dir ${ARTIFACTS_PATH}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)

compiler/circle-interpreter-cffi-test/README.md

@@ -0,0 +1,11 @@
+# circle-interpreter-cffi-test
+
+The `circle_interpereter_cffi` library wrapped with CFFI is designed to expose an existing `luci-interpreter` class to Python. It simplifies the integration of the class by creating a Python-compatible interface using CFFI.
+
+`circle-interpreter-cffi-test` ensures that the Python bindings for the C++ library correctly. Specifically, it verifies that:
+
+1. The CFFI-wrapped library can succesfully load the circle model.
+2. Inputs passed from Python are correctly interpreted by the `luci-interpreter`.
+3. The output generated by the interpreter matches the expected results.
+
+This test provides confidence that `luci-interpter`, when accessed through the Python interface, produces the same results as the original implementation.

compiler/circle-interpreter-cffi-test/infer.py

@@ -0,0 +1,112 @@
+import argparse
+import h5py
+import numpy as np
+from pathlib import Path
+import re
+import sys
+
+############ Managing paths for the artifacts required by the test.
+
+
+def extract_test_args(s):
+    p = re.compile('eval\\((.*)\\)')
+    result = p.search(s)
+    return result.group(1)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--lib_path', type=str, required=True)
+parser.add_argument('--test_list', type=str, required=True)
+parser.add_argument('--artifact_dir', type=str, required=True)
+args = parser.parse_args()
+
+with open(args.test_list) as f:
+    contents = [line.rstrip() for line in f]
+# remove newline and comments.
+eval_lines = [line for line in contents if line.startswith('eval(')]
+test_args = [extract_test_args(line) for line in eval_lines]
+test_models = [Path(args.artifact_dir) / f'{arg}.circle' for arg in test_args]
+input_data = [
+    Path(args.artifact_dir) / f'{arg}.opt/metadata/tc/input.h5' for arg in test_args
+]
+expected_output_data = [
+    Path(args.artifact_dir) / f'{arg}.opt/metadata/tc/expected.h5' for arg in test_args
+]
+
+############ CFFI test
+
+from cffi import FFI
+
+ffi = FFI()
+ffi.cdef("""
+  typedef struct InterpreterWrapper InterpreterWrapper;
+
+  const char *get_last_error(void);
+  void clear_last_error(void);
+  InterpreterWrapper *Interpreter_new(const uint8_t *data, const size_t data_size);
+  void Interpreter_delete(InterpreterWrapper *intp);
+  void Interpreter_interpret(InterpreterWrapper *intp);
+  void Interpreter_writeInputTensor(InterpreterWrapper *intp, const int input_idx, const void *data, size_t input_size);
+  void Interpreter_readOutputTensor(InterpreterWrapper *intp, const int output_idx, void *output, size_t output_size);
+""")
+C = ffi.dlopen(args.lib_path)
+
+
+def check_for_errors():
+    error_message = ffi.string(C.get_last_error()).decode('utf-8')
+    if error_message:
+        C.clear_last_error()
+        raise RuntimeError(f'C++ Exception: {error_message}')
+
+
+def error_checked(func):
+    """
+  Decorator to wrap functions with error checking.
+  """
+    def wrapper(*args, **kwargs):
+        result = func(*args, **kwargs)
+        check_for_errors()
+        return result
+
+    return wrapper
+
+
+Interpreter_new = error_checked(C.Interpreter_new)
+Interpreter_delete = error_checked(C.Interpreter_delete)
+Interpreter_interpret = error_checked(C.Interpreter_interpret)
+Interpreter_writeInputTensor = error_checked(C.Interpreter_writeInputTensor)
+Interpreter_readOutputTensor = error_checked(C.Interpreter_readOutputTensor)
+
+for idx, model_path in enumerate(test_models):
+    with open(model_path, "rb") as f:
+        model_data = ffi.from_buffer(bytearray(f.read()))
+
+    try:
+        intp = Interpreter_new(model_data, len(model_data))
+
+        # Set inputs
+        h5 = h5py.File(input_data[idx])
+        input_values = h5.get('value')
+        input_num = len(input_values)
+        for input_idx in range(input_num):
+            arr = np.array(input_values.get(str(input_idx)))
+            c_arr = ffi.from_buffer(arr)
+            Interpreter_writeInputTensor(intp, input_idx, c_arr, arr.nbytes)
+        # Do inference
+        Interpreter_interpret(intp)
+        # Check outputs
+        h5 = h5py.File(expected_output_data[idx])
+        output_values = h5.get('value')
+        output_num = len(output_values)
+        for output_idx in range(output_num):
+            arr = np.array(output_values.get(str(output_idx)))
+            result = np.empty(arr.shape, dtype=arr.dtype)
+            Interpreter_readOutputTensor(intp, output_idx, ffi.from_buffer(result),
+                                         arr.nbytes)
+            if not np.allclose(result, arr):
+                raise RuntimeError("Wrong outputs")
+
+        Interpreter_delete(intp)
+    except RuntimeError as e:
+        print(e)
+        sys.exit(-1)

compiler/circle-interpreter-cffi-test/requires.cmake

@@ -0,0 +1,2 @@
+require("common-artifacts")
+require("circle-interpreter")

compiler/circle-interpreter-cffi-test/test.lst

@@ -0,0 +1,17 @@
+eval(Add_000)
+eval(Add_U8_000)
+eval(AveragePool2D_000)
+eval(Concatenation_000)
+eval(Conv2D_000)
+eval(Conv2D_001)
+eval(Conv2D_002)
+eval(DepthwiseConv2D_000)
+eval(FullyConnected_000)
+eval(FullyConnected_001)
+eval(MaxPool2D_000)
+eval(Mul_000)
+eval(Pad_000)
+eval(Reshape_000)
+eval(Reshape_001)
+eval(Reshape_002)
+eval(Softmax_000)

compiler/circle-interpreter/CMakeLists.txt

@@ -11,3 +11,13 @@ target_link_libraries(circle-interpreter PRIVATE safemain)
 target_link_libraries(circle-interpreter PRIVATE vconone)
 
 install(TARGETS circle-interpreter DESTINATION bin)
+
+set(INTERPRETER_CFFI
+   src/CircleInterpreter_cffi.cpp
+)
+
+add_library(circle_interpreter_cffi SHARED ${INTERPRETER_CFFI})
+target_link_libraries(circle_interpreter_cffi PRIVATE luci_import)
+target_link_libraries(circle_interpreter_cffi PRIVATE luci_interpreter)
+
+install(TARGETS circle_interpreter_cffi DESTINATION lib)

compiler/circle-interpreter/src/CircleInterpreter_cffi.cpp

@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This library is for using luci-interpreter in Python. It is implemented
+ *  with CFFI. CFFI is a FFI (Foreign Function Interface) for Python calling
+ *  C code.
+ */
+
+#include <cstddef>
+#include <string>
+
+#include <luci/Importer.h>
+#include <luci_interpreter/Interpreter.h>
+
+namespace
+{
+
+// Global variable for error message
+std::string last_error_message;
+
+template <typename NodeT> size_t getTensorSize(const NodeT *node)
+{
+  uint32_t tensor_size = luci::size(node->dtype());
+  for (uint32_t i = 0; i < node->rank(); ++i)
+    tensor_size *= node->dim(i).value();
+  return tensor_size;
+}
+
+// Function to retrieve the last error message.
+extern "C" const char *get_last_error(void) { return last_error_message.c_str(); }
+
+// Clear the last error message
+extern "C" void clear_last_error(void) { last_error_message.clear(); }
+
+/**
+ * @brief A function that wraps another function and catches any exceptions.
+ *
+ * This function executes the given callable (`func`) with the provided arguments.
+ * If the callable throws an exception, the exception message is stored in a
+ *  `last_error_message` global variable.
+ *
+ * @tparam Func The type of the callable funciton.
+ * @tparam Args The types of arguments to pass to the callable function.
+ * @param func The callable function to execute.
+ * @param args The arguments to pass to the callable function.
+ * @return The return value of the callable function, or a default value in case of
+ *         an exception. If the function has a `void` return type, it simply returns
+ *         without any value.
+ *
+ * @note This function ensures that exceptions are safely caught and conveted to
+ *       error messages that can be queried externally, e.g. from Python.
+ */
+template <typename Func, typename... Args>
+auto exception_wrapper(Func func, Args... args) -> typename std::result_of<Func(Args...)>::type
+{
+  using ReturnType = typename std::result_of<Func(Args...)>::type;
+
+  try
+  {
+    return func(std::forward<Args>(args)...);
+  }
+  catch (const std::exception &e)
+  {
+    last_error_message = e.what();
+    if constexpr (not std::is_void<ReturnType>::value)
+    {
+      return ReturnType{};
+    }
+  }
+  catch (...)
+  {
+    last_error_message = "Unknown error";
+    if constexpr (not std::is_void<ReturnType>::value)
+    {
+      return ReturnType{};
+    }
+  }
+}
+
+} // namespace
+
+/*
+ * Q) Why do we need this wrapper class?
+ *
+ * A) This class is designed to address specific constraints introduced by
+ *   the use of CFFI for Python bindings. The original class lacked the
+ *   ability to maintain internal state like luci::Module because it is defined
+ *   out of the class. But, in Python, there's no way to keep luci::Module in memory.
+ *   To overcome this limitation, a wrapper class is implemented to manage and
+ *   keep states efficiently.
+ *
+ *   Moreover, the original interface relied on passing C++ classes as arguments,
+ *   which posed compatibility challenges when exposed to Python via CFFI. To
+ *   simplify the integration process, the wrapper class redesigns the interface
+ *   to accept more generic inputs, such as primitive types or standard containers,
+ *   ensuring seamless interaction between C++ and Python.
+ *
+ *   Overall, the redesigned class or the wrapper class preserves the original
+ *   functionality while introducing state management and a more flexible inteface,
+ *   making it highly suitable for Python-C++ interoperability through CFFI.
+ */
+class InterpreterWrapper
+{
+public:
+  explicit InterpreterWrapper(const uint8_t *data, const size_t data_size)
+  {
+    luci::Importer importer;
+    _module = importer.importModule(data, data_size);
+    if (_module == nullptr)
+    {
+      throw std::runtime_error{"Cannot import module."};
+    }
+    _intp = new luci_interpreter::Interpreter(_module.get());
+  }
+
+  ~InterpreterWrapper() { delete _intp; }
+
+  void interpret(void) { _intp->interpret(); }
+
+  void writeInputTensor(const int input_idx, const void *data, size_t input_size)
+  {
+    const auto input_nodes = loco::input_nodes(_module->graph());
+    const auto input_node = loco::must_cast<const luci::CircleInput *>(input_nodes.at(input_idx));
+    // Input size from model binary
+    const auto fb_input_size = ::getTensorSize(input_node);
+    if (fb_input_size != input_size)
+    {
+      const auto msg = "Invalid input size: " + std::to_string(fb_input_size) +
+                       " != " + std::to_string(input_size);
+      throw std::runtime_error(msg);
+    }
+    _intp->writeInputTensor(input_node, data, fb_input_size);
+  }
+
+  void readOutputTensor(const int output_idx, void *output, size_t output_size)
+  {
+    const auto output_nodes = loco::output_nodes(_module->graph());
+    const auto output_node =
+      loco::must_cast<const luci::CircleOutput *>(output_nodes.at(output_idx));
+    const auto fb_output_size = ::getTensorSize(output_node);
+    if (fb_output_size != output_size)
+    {
+      const auto msg = "Invalid output size: " + std::to_string(fb_output_size) +
+                       " != " + std::to_string(output_size);
+      throw std::runtime_error(msg);
+    }
+    _intp->readOutputTensor(output_node, output, fb_output_size);
+  }
+
+private:
+  luci_interpreter::Interpreter *_intp = nullptr;
+  std::unique_ptr<luci::Module> _module;
+};
+
+/*
+ * CFFI primarily uses functions instead of classes because it is designed to
+ *  work with C-compatible interfaces.
+ *
+ * - This extern "C" is necessary to avoid name mangling.
+ * - Explicitly pass the object pointer to any funcitons that operates on the object.
+ */
+extern "C" {
+
+InterpreterWrapper *Interpreter_new(const uint8_t *data, const size_t data_size)
+{
+  return ::exception_wrapper([&]() { return new InterpreterWrapper(data, data_size); });
+}
+
+void Interpreter_delete(InterpreterWrapper *intp)
+{
+  ::exception_wrapper([&]() { delete intp; });
+}
+
+void Interpreter_interpret(InterpreterWrapper *intp)
+{
+  ::exception_wrapper([&]() { intp->interpret(); });
+}
+
+void Interpreter_writeInputTensor(InterpreterWrapper *intp, const int input_idx, const void *data,
+                                  size_t input_size)
+{
+  ::exception_wrapper([&]() { intp->writeInputTensor(input_idx, data, input_size); });
+}
+
+void Interpreter_readOutputTensor(InterpreterWrapper *intp, const int output_idx, void *output,
+                                  size_t output_size)
+{
+  ::exception_wrapper([&]() { intp->readOutputTensor(output_idx, output, output_size); });
+}
+
+} // extern "C"

compiler/common-artifacts/CMakeLists.txt

@@ -76,6 +76,7 @@ if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
     COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
     COMMAND ${CMAKE_COMMAND} -E echo "pytest==7.4.3" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
     COMMAND ${CMAKE_COMMAND} -E echo "h5py==3.11.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
+    COMMAND ${CMAKE_COMMAND} -E echo "cffi==1.16.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
     COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
             ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
     COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
@@ -92,6 +93,7 @@ else(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
     COMMAND ${CMAKE_COMMAND} -E echo "pydot==1.4.2" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
     COMMAND ${CMAKE_COMMAND} -E echo "pytest==7.4.3" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
     COMMAND ${CMAKE_COMMAND} -E echo "h5py==3.11.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
+    COMMAND ${CMAKE_COMMAND} -E echo "cffi==1.16.0" >> ${REQUIREMENTS_OVERLAY_PATH_TF_2_12_1}
     COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000
             ${PIP_OPTION_TRUSTED_HOST} install --upgrade pip setuptools
     COMMAND ${VIRTUALENV_OVERLAY_TF_2_12_1}/bin/${PYTHON_OVERLAY} -m pip --default-timeout=1000