Skip to content

Commit ad40d76

Browse files
seanshparkseanshpark
committed
DRAFT initial circle-mlir project
on-going draft to introduce initial circle-mlir project. Signed-off-by: SaeHie Park <[email protected]>
1 parent 5a67df1 commit ad40d76

28 files changed

+2301
-4
lines changed

‎.github/workflows/run-circle-mlir-build.yml

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ jobs:
3535
include:
3636
- ubuntu_code: jammy
3737
ubuntu_vstr: u2204
38+
one_comp_ver: 1.29.0
3839

3940
runs-on: ubuntu-latest
4041

@@ -45,11 +46,19 @@ jobs:
4546
name: circle-mlir ${{ matrix.ubuntu_vstr }} ${{ matrix.type }} test
4647

4748
steps:
49+
# TODO prepare circle-interpreter Debian package and install
50+
- name: Install one-compiler
51+
run: |
52+
cd /var/tmp
53+
ONE_COMPILER=one-compiler-${{ matrix.ubuntu_code }}_${{ matrix.one_comp_ver }}_amd64.deb
54+
wget https://github.com/Samsung/ONE/releases/download/${{ matrix.one_comp_ver }}/${ONE_COMPILER}
55+
ls -al .
56+
dpkg -i ${ONE_COMPILER}
57+
ls -al /usr/share/one/bin
58+
4859
- name: Checkout
4960
uses: actions/checkout@v4
5061

51-
# TODO download circle-interpreter
52-
5362
# NOTE Docker image has pre-installed submodules in /workdir
5463
# NOTE Docker image has pre-installed python packages
5564
- name: Configure

‎circle-mlir/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
Makefile
22
build
3+
infra/overlay/venv

‎circle-mlir/Makefile.sample

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# find python3 from overlay or else from PATH
2+
PYTHON3_CMD:=./infra/overlay/venv/bin/python3
3+
ifeq ($(wildcard $(PYTHON3_CMD)),)
4+
PYTHON3_CMD:=$(shell which python3)
5+
endif
6+
# TODO error handle if not found
7+
PYTHON3_PATH=$(shell dirname $(PYTHON3_CMD))
8+
9+
CIRCLEMLIR_BUILD_DEBUG?=build/debug
10+
CIRCLEMLIR_BUILD_REL?=build/release
11+
12+
CIRCLEMLIR_PY3_ROOT?=$(PYTHON3_PATH)
13+
CIRCLEMLIR_BUILD_JOBS?=4
14+
15+
.PHONY: help all overlay \
16+
prep cfg debug test clean \
17+
prepr cfgr rel testr cleanr \
18+
_mkbuild _mkbuildr
19+
20+
.DEFAULT_GOAL: help
21+
22+
help:
23+
@echo "make overlay : prepare overlay (needed only once)"
24+
@echo "make prep : prepare submodules for debug (needed only once)"
25+
@echo "make cfg : configure circle-mlir for debug build"
26+
@echo "make debug : build for debug"
27+
@echo "make test : test for debug"
28+
@echo "make clean : clean debug build"
29+
@echo "make prepr : prepare submodules for release (needed only once)"
30+
@echo "make cfgr : configure circle-mlir for release build"
31+
@echo "make rel : build for release"
32+
@echo "make testr : test for release"
33+
@echo "make cleanr : clean release build"
34+
@echo "make cleanall : clean all build including overlay, submodules"
35+
36+
all: cfg debug
37+
38+
#-------------------------------------------------------------------------------
39+
# targets for internal usage
40+
#
41+
_mkbuild:
42+
mkdir -p $(CIRCLEMLIR_BUILD_DEBUG)
43+
44+
_mkbuildr:
45+
mkdir -p $(CIRCLEMLIR_BUILD_REL)
46+
47+
#-------------------------------------------------------------------------------
48+
# targets for users
49+
#
50+
overlay:
51+
bash infra/overlay/prepare-venv
52+
53+
prep: _mkbuild
54+
Python3_ROOT_DIR=$(CIRCLEMLIR_PY3_ROOT) \
55+
cmake -B $(CIRCLEMLIR_BUILD_DEBUG)/externals -S ./externals -DCMAKE_BUILD_TYPE=Debug
56+
cmake --build $(CIRCLEMLIR_BUILD_DEBUG)/externals -j$(CIRCLEMLIR_BUILD_JOBS)
57+
58+
cfg: _mkbuild
59+
cmake -B $(CIRCLEMLIR_BUILD_DEBUG) -S ./ \
60+
-DONNX2CIRCLE_TEST_MODELS_SINGLE=ON
61+
62+
debug:
63+
CM_PASS_DUMP=1 \
64+
cmake --build $(CIRCLEMLIR_BUILD_DEBUG) -j$(CIRCLEMLIR_BUILD_JOBS)
65+
66+
test:
67+
CTEST_OUTPUT_ON_FAILURE=1 \
68+
cmake --build $(CIRCLEMLIR_BUILD_DEBUG) --verbose -- test
69+
70+
clean:
71+
rm -f $(CIRCLEMLIR_BUILD_DEBUG)/CMakeCache.txt
72+
rm -rf $(CIRCLEMLIR_BUILD_DEBUG)/circle-mlir/
73+
74+
#-------------------------------------------------------------------------------
75+
# for release
76+
77+
prepr: _mkbuildr
78+
Python3_ROOT_DIR=$(CIRCLEMLIR_PY3_ROOT) \
79+
cmake -B $(CIRCLEMLIR_BUILD_REL)/submodules -S ./submodules -DCMAKE_BUILD_TYPE=Release
80+
cmake --build $(CIRCLEMLIR_BUILD_REL)/submodules -j$(CIRCLEMLIR_BUILD_JOBS)
81+
82+
cfgr: _mkbuildr
83+
cmake -B $(CIRCLEMLIR_BUILD_REL) -S ./ -DCMAKE_BUILD_TYPE=Release
84+
85+
rel:
86+
cmake --build $(CIRCLEMLIR_BUILD_REL) -j$(CIRCLEMLIR_BUILD_JOBS)
87+
88+
testr:
89+
CTEST_OUTPUT_ON_FAILURE=1 \
90+
cmake --build $(CIRCLEMLIR_BUILD_REL) --verbose -- test
91+
92+
cleanr:
93+
rm -f $(CIRCLEMLIR_BUILD_REL)/CMakeCache.txt
94+
rm -rf $(CIRCLEMLIR_BUILD_REL)/circle-mlir/
95+
96+
97+
#-------------------------------------------------------------------------------
98+
99+
cleanall:
100+
rm -rf $(CIRCLEMLIR_BUILD_DEBUG)
101+
rm -rf $(CIRCLEMLIR_BUILD_REL)
102+
rm -rf infra/overlay/venv

‎circle-mlir/README.md

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,175 @@
11
# circle-mlir
22

33
Circle MLIR dialect and tools
4+
5+
## Tools provided
6+
7+
_onnx2circle_
8+
- conversion tool of ONNX to Circle model for `compiler`
9+
- to replace not-maintained-anymore onnx-tensorflow package
10+
11+
## How to build
12+
13+
Use provided `Makefile.sample` or create your own `Makefile`
14+
```
15+
ln -s Makefile.sample Makefile
16+
```
17+
- `Makefile` is in `.gitignore` to let developers use own Makefile.
18+
19+
### Prerequisite
20+
21+
```
22+
sudo apt-get install build-essential cmake git fakeroot
23+
sudo apt-get install autoconf automake libtool unzip wget
24+
sudo apt-get install devscripts debmake debhelper lcov
25+
sudo apt-get install python3 python3-pip python3-venv python3-dev python3-all dh-python
26+
27+
python3 -m pip install --upgrade pip setuptools
28+
python3 -m pip install numpy h5py==3.8.0
29+
```
30+
31+
### Prepare externals
32+
33+
### Debug build
34+
35+
Prepare overlay
36+
```
37+
make overlay
38+
```
39+
40+
Build submodules in venv
41+
```
42+
source infra/overlay/venv/bin/activate
43+
make prep
44+
```
45+
NOTE `llvm-project` is built as `Debug` which may require 32G or more RAM.
46+
- if build fails for some reason, please change back to
47+
`-DCMAKE_BUILD_TYPE=Release` in `prep:` target in `Makefile.sample` file.
48+
- build and test needs venv python packages.
49+
50+
NOTE `overlay` and `submodules` builds are needed only once.
51+
52+
Configure and build
53+
```
54+
make cfg
55+
make debug
56+
```
57+
58+
Test build
59+
```
60+
make test
61+
```
62+
- optionally, set `ONE_COMPILER_ROOT` to alternate PATH for local ONE build
63+
```
64+
ONE_COMPILER_ROOT=/home/user/one/build/install make test
65+
```
66+
67+
To clean up existing build results
68+
```
69+
make clean
70+
```
71+
72+
To clean up also `overlay` and `submodules`
73+
```
74+
make cleanall
75+
```
76+
- NOTE when using `CIRCLE_MLIR_LOCALINST`, need to manually clean up this folder
77+
78+
### Release build
79+
80+
Release build is available as follows.
81+
Others not mentioned are same as above Debug build.
82+
83+
Build submodules in venv
84+
```
85+
source infra/overlay/venv/bin/activate
86+
make prepr
87+
deactivate
88+
```
89+
90+
Configure and build
91+
```
92+
make cfgr
93+
make rel
94+
```
95+
96+
Test build
97+
```
98+
make testr
99+
```
100+
101+
### Test coverage
102+
103+
To get test coverage report, run as following commands.
104+
- assume you already have done `make overlay` and `make prepcov`
105+
- you can skip `make prepcov` step if you are using local installation with `CIRCLE_MLIR_LOCALINST`
106+
- or you can reuse `CIRCLE_MLIR_LOCALINST` for existing debug or release build submodules with
107+
`cfgcov` target such as `CIRCLE_MLIR_LOCALINST=$(pwd)/build/debug/submodules make cfgcov`
108+
```
109+
source infra/overlay/venv/bin/activate
110+
make cfgcov
111+
deactivate
112+
113+
make debugcov
114+
make testcov
115+
make gencov
116+
```
117+
118+
Open `converage/html/index.html` file in web browser to see the reports.
119+
120+
To generate from second run and so on in your local machine, you will have to
121+
remove existing files before running `gencov`
122+
```
123+
rm -rf coverage
124+
make gencov
125+
```
126+
127+
To run this with Docker image, use `cfgcovdi` target instead of `cfgcov`.
128+
```
129+
make cfgcovdi
130+
make debugcov
131+
make testcov
132+
make gencov
133+
```
134+
135+
136+
## Local format check
137+
138+
Install prerequiste package.
139+
```
140+
sudo apt-get install clang-format-12 python3 python3-pip
141+
python3 -m pip install yapf==0.32.0
142+
```
143+
144+
Run format checker.
145+
```
146+
bash ./infra/tools/format
147+
```
148+
or with `Makefile` from `Makefile.sample`
149+
```
150+
make format
151+
```
152+
153+
## Dump debug logs
154+
155+
To see logs during conversion with `onnx2circle` tool, set `CM_PASS_DUMP=1` for
156+
preprocessing ONNX and ONNX to circle conversion, or set `CM_PASS_DUMP=2` to see
157+
additional logs for circle rewrite.
158+
159+
```
160+
CM_PASS_DUMP=2 onnx2circle input.onnx output.circle
161+
```
162+
163+
You can give `-debug` option to see general MLIR logs or `-debug-only=o2c`
164+
option to see only logs from onnx2circle.
165+
166+
```
167+
onnx2circle -debug-only=o2c input.onnx output.circle
168+
```
169+
170+
## TensorFlow source code
171+
172+
Some source codes are referenced from TensorFlow and the file path is added to
173+
inside our source.
174+
175+
Current codes are from `v2.12.1` tag.

‎circle-mlir/circle-mlir/CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
include(UseMLIR)
2+
include(UseAbseil)
3+
24
add_subdirectory(lib)
5+
add_subdirectory(tools)
36
add_subdirectory(tools-test)

‎circle-mlir/circle-mlir/lib/CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,6 @@ add_subdirectory(arser)
44
add_subdirectory(schema)
55
add_subdirectory(dialect)
66
add_subdirectory(utils)
7+
#add_subdirectory(pass)
8+
#add_subdirectory(import)
9+
add_subdirectory(export)

‎circle-mlir/circle-mlir/lib/dialect/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ target_include_directories(cirmlir_dialect PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/in
1818
# use generated files
1919
add_dependencies(cirmlir_dialect circle_mlir_gen_inc)
2020
target_include_directories(cirmlir_dialect PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
21+
target_link_libraries(cirmlir_dialect PUBLIC abseil_cpp)
2122
target_link_libraries(cirmlir_dialect PUBLIC circle_schema)
2223
target_link_libraries(cirmlir_dialect PUBLIC cirmlir_coverage)
2324

0 commit comments

Comments
 (0)