DRAFT initial circle-mlir project

on-going draft to introduce initial circle-mlir project.

Signed-off-by: SaeHie Park <[email protected]>

.github/workflows/run-circle-mlir-build.yml

@@ -35,6 +35,7 @@ jobs:
         include:
           - ubuntu_code: jammy
             ubuntu_vstr: u2204
+            one_comp_ver: 1.29.0
 
     runs-on: ubuntu-latest
 
@@ -45,11 +46,19 @@ jobs:
     name: circle-mlir ${{ matrix.ubuntu_vstr }} ${{ matrix.type }} test
 
     steps:
+      # TODO prepare circle-interpreter Debian package and install
+      - name: Install one-compiler
+        run: |
+          cd /var/tmp
+          ONE_COMPILER=one-compiler-${{ matrix.ubuntu_code }}_${{ matrix.one_comp_ver }}_amd64.deb
+          wget https://github.com/Samsung/ONE/releases/download/${{ matrix.one_comp_ver }}/${ONE_COMPILER}
+          ls -al .
+          dpkg -i ${ONE_COMPILER}
+          ls -al /usr/share/one/bin
+
       - name: Checkout
         uses: actions/checkout@v4
 
-      # TODO download circle-interpreter
-
       # NOTE Docker image has pre-installed submodules in /workdir
       # NOTE Docker image has pre-installed python packages
       - name: Configure

circle-mlir/.gitignore

@@ -1,2 +1,3 @@
 Makefile
 build
+infra/overlay/venv

circle-mlir/Makefile.sample

@@ -0,0 +1,102 @@
+# find python3 from overlay or else from PATH
+PYTHON3_CMD:=./infra/overlay/venv/bin/python3
+ifeq ($(wildcard $(PYTHON3_CMD)),)
+	PYTHON3_CMD:=$(shell which python3)
+endif
+# TODO error handle if not found
+PYTHON3_PATH=$(shell dirname $(PYTHON3_CMD))
+
+CIRCLEMLIR_BUILD_DEBUG?=build/debug
+CIRCLEMLIR_BUILD_REL?=build/release
+
+CIRCLEMLIR_PY3_ROOT?=$(PYTHON3_PATH)
+CIRCLEMLIR_BUILD_JOBS?=4
+
+.PHONY: help all overlay \
+	prep cfg debug test clean \
+	prepr cfgr rel testr cleanr \
+	_mkbuild _mkbuildr
+
+.DEFAULT_GOAL: help
+
+help:
+	@echo "make overlay  : prepare overlay (needed only once)"
+	@echo "make prep     : prepare submodules for debug (needed only once)"
+	@echo "make cfg      : configure circle-mlir for debug build"
+	@echo "make debug    : build for debug"
+	@echo "make test     : test for debug"
+	@echo "make clean    : clean debug build"
+	@echo "make prepr    : prepare submodules for release (needed only once)"
+	@echo "make cfgr     : configure circle-mlir for release build"
+	@echo "make rel      : build for release"
+	@echo "make testr    : test for release"
+	@echo "make cleanr   : clean release build"
+	@echo "make cleanall : clean all build including overlay, submodules"
+
+all: cfg debug
+
+#-------------------------------------------------------------------------------
+# targets for internal usage
+#
+_mkbuild:
+	mkdir -p $(CIRCLEMLIR_BUILD_DEBUG)
+
+_mkbuildr:
+	mkdir -p $(CIRCLEMLIR_BUILD_REL)
+
+#-------------------------------------------------------------------------------
+# targets for users
+#
+overlay:
+	bash infra/overlay/prepare-venv
+
+prep: _mkbuild
+	Python3_ROOT_DIR=$(CIRCLEMLIR_PY3_ROOT) \
+	cmake -B $(CIRCLEMLIR_BUILD_DEBUG)/externals -S ./externals -DCMAKE_BUILD_TYPE=Debug
+	cmake --build $(CIRCLEMLIR_BUILD_DEBUG)/externals -j$(CIRCLEMLIR_BUILD_JOBS)
+
+cfg: _mkbuild
+	cmake -B $(CIRCLEMLIR_BUILD_DEBUG) -S ./ \
+	-DONNX2CIRCLE_TEST_MODELS_SINGLE=ON
+
+debug:
+	CM_PASS_DUMP=1 \
+	cmake --build $(CIRCLEMLIR_BUILD_DEBUG) -j$(CIRCLEMLIR_BUILD_JOBS)
+
+test:
+	CTEST_OUTPUT_ON_FAILURE=1 \
+	cmake --build $(CIRCLEMLIR_BUILD_DEBUG) --verbose -- test
+
+clean:
+	rm -f $(CIRCLEMLIR_BUILD_DEBUG)/CMakeCache.txt
+	rm -rf $(CIRCLEMLIR_BUILD_DEBUG)/circle-mlir/
+
+#-------------------------------------------------------------------------------
+# for release
+
+prepr: _mkbuildr
+	Python3_ROOT_DIR=$(CIRCLEMLIR_PY3_ROOT) \
+	cmake -B $(CIRCLEMLIR_BUILD_REL)/submodules -S ./submodules -DCMAKE_BUILD_TYPE=Release
+	cmake --build $(CIRCLEMLIR_BUILD_REL)/submodules -j$(CIRCLEMLIR_BUILD_JOBS)
+
+cfgr: _mkbuildr
+	cmake -B $(CIRCLEMLIR_BUILD_REL) -S ./ -DCMAKE_BUILD_TYPE=Release
+
+rel:
+	cmake --build $(CIRCLEMLIR_BUILD_REL) -j$(CIRCLEMLIR_BUILD_JOBS)
+
+testr:
+	CTEST_OUTPUT_ON_FAILURE=1 \
+	cmake --build $(CIRCLEMLIR_BUILD_REL) --verbose -- test
+
+cleanr:
+	rm -f $(CIRCLEMLIR_BUILD_REL)/CMakeCache.txt
+	rm -rf $(CIRCLEMLIR_BUILD_REL)/circle-mlir/
+
+
+#-------------------------------------------------------------------------------
+
+cleanall:
+	rm -rf $(CIRCLEMLIR_BUILD_DEBUG)
+	rm -rf $(CIRCLEMLIR_BUILD_REL)
+	rm -rf infra/overlay/venv

circle-mlir/README.md

@@ -1,3 +1,175 @@
 # circle-mlir
 
 Circle MLIR dialect and tools
+
+## Tools provided
+
+_onnx2circle_
+- conversion tool of ONNX to Circle model for `compiler`
+- to replace not-maintained-anymore onnx-tensorflow package
+
+## How to build
+
+Use provided `Makefile.sample` or create your own `Makefile`
+```
+ln -s Makefile.sample Makefile
+```
+- `Makefile` is in `.gitignore` to let developers use own Makefile.
+
+### Prerequisite
+
+```
+sudo apt-get install build-essential cmake git fakeroot
+sudo apt-get install autoconf automake libtool unzip wget
+sudo apt-get install devscripts debmake debhelper lcov
+sudo apt-get install python3 python3-pip python3-venv python3-dev python3-all dh-python
+
+python3 -m pip install --upgrade pip setuptools
+python3 -m pip install numpy h5py==3.8.0
+```
+
+### Prepare externals
+
+### Debug build
+
+Prepare overlay
+```
+make overlay
+```
+
+Build submodules in venv
+```
+source infra/overlay/venv/bin/activate
+make prep
+```
+NOTE `llvm-project` is built as `Debug` which may require 32G or more RAM.
+- if build fails for some reason, please change back to
+  `-DCMAKE_BUILD_TYPE=Release` in `prep:` target in `Makefile.sample` file.
+- build and test needs venv python packages.
+
+NOTE `overlay` and `submodules` builds are needed only once.
+
+Configure and build
+```
+make cfg
+make debug
+```
+
+Test build
+```
+make test
+```
+- optionally, set `ONE_COMPILER_ROOT` to alternate PATH for local ONE build
+  ```
+  ONE_COMPILER_ROOT=/home/user/one/build/install make test
+  ```
+
+To clean up existing build results
+```
+make clean
+```
+
+To clean up also `overlay` and `submodules`
+```
+make cleanall
+```
+- NOTE when using `CIRCLE_MLIR_LOCALINST`, need to manually clean up this folder
+
+### Release build
+
+Release build is available as follows.
+Others not mentioned are same as above Debug build.
+
+Build submodules in venv
+```
+source infra/overlay/venv/bin/activate
+make prepr
+deactivate
+```
+
+Configure and build
+```
+make cfgr
+make rel
+```
+
+Test build
+```
+make testr
+```
+
+### Test coverage
+
+To get test coverage report, run as following commands.
+- assume you already have done `make overlay` and `make prepcov`
+- you can skip `make prepcov` step if you are using local installation with `CIRCLE_MLIR_LOCALINST`
+- or you can reuse `CIRCLE_MLIR_LOCALINST` for existing debug or release build submodules with 
+`cfgcov` target such as `CIRCLE_MLIR_LOCALINST=$(pwd)/build/debug/submodules make cfgcov`
+```
+source infra/overlay/venv/bin/activate
+make cfgcov
+deactivate
+
+make debugcov
+make testcov
+make gencov
+```
+
+Open `converage/html/index.html` file in web browser to see the reports.
+
+To generate from second run and so on in your local machine, you will have to
+remove existing files before running `gencov`
+```
+rm -rf coverage
+make gencov
+```
+
+To run this with Docker image, use `cfgcovdi` target instead of `cfgcov`.
+```
+make cfgcovdi
+make debugcov
+make testcov
+make gencov
+```
+
+
+## Local format check
+
+Install prerequiste package.
+```
+sudo apt-get install clang-format-12 python3 python3-pip
+python3 -m pip install yapf==0.32.0
+```
+
+Run format checker.
+```
+bash ./infra/tools/format
+```
+or with `Makefile` from `Makefile.sample`
+```
+make format
+```
+
+## Dump debug logs
+
+To see logs during conversion with `onnx2circle` tool, set `CM_PASS_DUMP=1` for
+preprocessing ONNX and ONNX to circle conversion, or set `CM_PASS_DUMP=2` to see
+additional logs for circle rewrite.
+
+```
+CM_PASS_DUMP=2 onnx2circle input.onnx output.circle
+```
+
+You can give `-debug` option to see general MLIR logs or `-debug-only=o2c`
+option to see only logs from onnx2circle.
+
+```
+onnx2circle -debug-only=o2c input.onnx output.circle
+```
+
+## TensorFlow source code
+
+Some source codes are referenced from TensorFlow and the file path is added to
+inside our source.
+
+Current codes are from `v2.12.1` tag.

circle-mlir/circle-mlir/CMakeLists.txt

@@ -1,3 +1,6 @@
 include(UseMLIR)
+include(UseAbseil)
+
 add_subdirectory(lib)
+add_subdirectory(tools)
 add_subdirectory(tools-test)

circle-mlir/circle-mlir/lib/CMakeLists.txt

@@ -4,3 +4,6 @@ add_subdirectory(arser)
 add_subdirectory(schema)
 add_subdirectory(dialect)
 add_subdirectory(utils)
+#add_subdirectory(pass)
+#add_subdirectory(import)
+add_subdirectory(export)

circle-mlir/circle-mlir/lib/dialect/CMakeLists.txt

@@ -18,6 +18,7 @@ target_include_directories(cirmlir_dialect PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/in
 # use generated files
 add_dependencies(cirmlir_dialect circle_mlir_gen_inc)
 target_include_directories(cirmlir_dialect PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+target_link_libraries(cirmlir_dialect PUBLIC abseil_cpp)
 target_link_libraries(cirmlir_dialect PUBLIC circle_schema)
 target_link_libraries(cirmlir_dialect PUBLIC cirmlir_coverage)