Skip to content

Commit 9d2b2ef

Browse files
KOLANICHKOLANICH
committed
Initial commit
0 parents  commit 9d2b2ef

34 files changed

+2374
-0
lines changed

‎.editorconfig

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
root = true
2+
3+
[*]
4+
charset = utf-8
5+
indent_style = tab
6+
indent_size = 4
7+
insert_final_newline = true
8+
end_of_line = lf
9+
10+
[*.{yml,yaml,ksy}]
11+
indent_style = space
12+
indent_size = 2

‎.github/.templateMarker

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
KOLANICH/python_project_boilerplate.py

‎.github/dependabot.yml

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: "pip"
4+
directory: "/"
5+
schedule:
6+
interval: "daily"
7+
allow:
8+
- dependency-type: "all"

‎.github/workflows/CI.yml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
name: CI
2+
on:
3+
push:
4+
branches: [master]
5+
pull_request:
6+
branches: [master]
7+
8+
jobs:
9+
build:
10+
runs-on: ubuntu-22.04
11+
steps:
12+
- name: typical python workflow
13+
uses: KOLANICH-GHActions/typical-python-workflow@master
14+
with:
15+
github_token: ${{ secrets.GITHUB_TOKEN }}

‎.gitignore

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
__pycache__
2+
*.pyc
3+
*.pyo
4+
/kaitaiStructCompile.egg-info
5+
/build
6+
/dist
7+
/.eggs
8+
/tests/output
9+
/tests/formats

‎.gitlab-ci.yml

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
image: registry.gitlab.com/kaitaistructcompile.py/kaitai_struct_python_docker:latest
2+
3+
stages:
4+
- build
5+
- test
6+
- tooling
7+
8+
variables:
9+
GIT_DEPTH: "1"
10+
DOCKER_DRIVER: overlay2
11+
SAST_ANALYZER_IMAGE_TAG: latest
12+
SAST_DISABLE_DIND: "true"
13+
SAST_CONFIDENCE_LEVEL: 5
14+
CODECLIMATE_VERSION: latest
15+
16+
include:
17+
- template: SAST.gitlab-ci.yml
18+
- template: Code-Quality.gitlab-ci.yml
19+
20+
build:
21+
tags:
22+
- shared
23+
- linux
24+
stage: build
25+
variables:
26+
GIT_SUBMODULE_STRATEGY: recursive
27+
PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages
28+
29+
before_script:
30+
- export PATH="$PATH:$PYTHONUSERBASE/bin" # don't move into `variables`
31+
- git clone --depth=1 https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.tests.ksys ./tests/ksys
32+
33+
cache:
34+
paths:
35+
- $PYTHONUSERBASE
36+
- tests/kaitai_struct_formats
37+
38+
script:
39+
- python3 setup.py bdist_wheel
40+
- pip3 install --upgrade --pre ./dist/*.whl
41+
- mkdir ./wheels
42+
- mv ./dist/*.whl ./wheels/kaitaiStructCompile-0.CI-py3-none-any.whl
43+
- cd ./tests # in order to isolate from kaitaiStructCompile dir in current dir, which doesn't contain backends, so interferes with them.
44+
- coverage run --source=kaitaiStructCompile --branch -m pytest --junitxml=./rspec.xml ./test.py
45+
- coverage report -m
46+
- coverage xml
47+
48+
coverage: /^TOTAL(?:\s*\d+){4}\s(\d+%).+/
49+
50+
artifacts:
51+
paths:
52+
- wheels
53+
- tests/output
54+
reports:
55+
junit: tests/rspec.xml
56+
cobertura: ./coverage.xml
57+
58+
59+
.pages: # disabled, remove dot to enable
60+
stage: tooling
61+
tags:
62+
- shared
63+
- linux
64+
image: alpine:latest
65+
allow_failure: true
66+
before_script:
67+
- apk update
68+
- apk add doxygen
69+
- apk add ttf-freefont graphviz
70+
script:
71+
- doxygen ./Doxyfile
72+
- mv ./docs/html ./public
73+
artifacts:
74+
paths:
75+
- public
76+
only:
77+
- master

‎Code_Of_Conduct.md

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
No codes of conduct!

‎Doxyfile

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
PROJECT_NAME = "kaitaiStructCompile.py"
2+
PROJECT_BRIEF = "the package to compile .ksy into .py while installing/building a python package"
3+
PROJECT_LOGO = "https://assets.gitlab-static.net/uploads/-/system/project/avatar/8079746/logo1536.jpg?width=40"
4+
5+
# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
6+
# into which the generated documentation will be written. If a relative path is
7+
# entered, it will be relative to the location where doxygen was started. If
8+
# left blank the current directory will be used.
9+
10+
OUTPUT_DIRECTORY = "./docs"
11+
ALLOW_UNICODE_NAMES = YES
12+
OUTPUT_LANGUAGE = English
13+
BRIEF_MEMBER_DESC = YES
14+
REPEAT_BRIEF = YES
15+
INLINE_INHERITED_MEMB = NO
16+
FULL_PATH_NAMES = YES
17+
STRIP_FROM_PATH =
18+
SHORT_NAMES = NO
19+
INHERIT_DOCS = YES
20+
SEPARATE_MEMBER_PAGES = NO
21+
TAB_SIZE = 4
22+
OPTIMIZE_OUTPUT_JAVA = YES
23+
MARKDOWN_SUPPORT = YES
24+
AUTOLINK_SUPPORT = YES
25+
IDL_PROPERTY_SUPPORT = YES
26+
27+
EXTRACT_ALL = YES
28+
RECURSIVE = YES
29+
30+
EXAMPLE_PATH =
31+
EXAMPLE_RECURSIVE = YES
32+
33+
USE_MDFILE_AS_MAINPAGE = ReadMe.md
34+
35+
SOURCE_BROWSER = YES
36+
INLINE_SOURCES = YES
37+
38+
39+
REFERENCED_BY_RELATION = YES
40+
REFERENCES_RELATION = YES
41+
VERBATIM_HEADERS = NO
42+
43+
ALPHABETICAL_INDEX = YES
44+
COLS_IN_ALPHA_INDEX = 5
45+
46+
GENERATE_HTML = YES
47+
GENERATE_LATEX = NO
48+
#HTML_OUTPUT = html
49+
HTML_FILE_EXTENSION = .html
50+
HTML_DYNAMIC_SECTIONS = YES
51+
HTML_INDEX_NUM_ENTRIES = 9999
52+
GENERATE_TREEVIEW = YES
53+
ENUM_VALUES_PER_LINE = 1
54+
EXT_LINKS_IN_WINDOW = NO
55+
FORMULA_FONTSIZE = 10
56+
FORMULA_TRANSPARENT = YES
57+
SEARCHENGINE = YES
58+
59+
#---------------------------------------------------------------------------
60+
# Configuration options related to the dot tool
61+
#---------------------------------------------------------------------------
62+
CLASS_DIAGRAMS = YES
63+
HIDE_UNDOC_RELATIONS = NO
64+
HAVE_DOT = YES
65+
DOT_FONTNAME = Helvetica
66+
DOT_FONTSIZE = 10
67+
UML_LOOK = YES
68+
TEMPLATE_RELATIONS = YES
69+
CALL_GRAPH = YES
70+
CALLER_GRAPH = YES
71+
GRAPHICAL_HIERARCHY = YES
72+
DIRECTORY_GRAPH = YES
73+
DOT_IMAGE_FORMAT = svg
74+
INTERACTIVE_SVG = YES
75+
DOT_GRAPH_MAX_NODES = 10000
76+
MAX_DOT_GRAPH_DEPTH = 0
77+
DOT_TRANSPARENT = YES
78+
DOT_MULTI_TARGETS = YES

‎MANIFEST.in

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
include UNLICENSE
2+
include *.md
3+
include tests
4+
include .editorconfig
5+
global-include *.json

‎ReadMe.md

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
kaitaiStructCompile.py [![Unlicensed work](https://raw.githubusercontent.com/unlicense/unlicense.org/master/static/favicon.png)](https://unlicense.org/)
2+
======================
3+
[wheel](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.py/-/jobs/artifacts/master/raw/wheels/kaitaiStructCompile-0.CI-py3-none-any.whl?job=build)
4+
[![PyPi Status](https://img.shields.io/pypi/v/kaitaiStructCompile.py.svg)](https://pypi.python.org/pypi/kaitaiStructCompile.py)
5+
[![GitLab build status](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.py/badges/master/pipeline.svg)](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.py/commits/master)
6+
[![Coveralls Coverage](https://img.shields.io/coveralls/KOLANICH/kaitaiStructCompile.py.svg)](https://coveralls.io/r/KOLANICH/kaitaiStructCompile.py)
7+
[![GitLab coverage](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.py/badges/master/coverage.svg)](https://gitlab.com/kaitaiStructCompile.py/kaitaiStructCompile.py/commits/master)
8+
[![Libraries.io Status](https://img.shields.io/librariesio/github/KOLANICH/kaitaiStructCompile.py.svg)](https://libraries.io/github/KOLANICH/kaitaiStructCompile.py)
9+
[![Code style: antiflash](https://img.shields.io/badge/code%20style-antiflash-FFF.svg)](https://github.com/KOLANICH-tools/antiflash.py)
10+
11+
This is a tool automating compilation [Kaitai Struct](https://github.com/kaitai-io/kaitai_struct) ```*.ksy``` files into python ones.
12+
13+
14+
Features
15+
--------
16+
17+
* python bindings to compile KS `.ksy` specs into python source code.
18+
* importer allowing to import `ksy`s. Seamlessly compiles `ksy`s into python sources. Useful for playing in IPython shell.
19+
* `setuptools` plugin to automate compiling `ksy`s into `py` in process of building of your package.
20+
21+
Prerequisites
22+
-------------
23+
* [Kaitai Struct compiler](https://github.com/kaitai-io/kaitai_struct_compiler) must be unpacked somewhere and all its prerequisites like `JRE` or `OpenJDK ` must be installed.
24+
* Path to Kaitai Struct compiler root dir (the one containing `lib`, `bin` and `formats`) must be exposed as `KAITAI_STRUCT_ROOT` environment variable.
25+
* A backend must be installed. [CLI backend](https://github.com/kaitaiStructCompile/kaitaiStructCompileBackendCLI) is currently the only one publicly available.
26+
27+
Usage
28+
-----
29+
30+
### Importing a ksy
31+
32+
```python
33+
import kaitaiStructCompile.importer
34+
kaitaiStructCompile.importer._importer.searchDirs.append(Path("./dirWithKSYFiles")) # you can add a dir to search for KSY files.
35+
kaitaiStructCompile.importer._importer.flags["readStoresPos"]=True # you can set compiler flags, for more details see the JSON schema
36+
from kaitaiStructCompile.importer.test import Test
37+
Test # kaitaiStructCompile.importer.test.Test
38+
```
39+
40+
### Manual compilation
41+
42+
```python
43+
from kaitaiStructCompile import compile
44+
compile("./ksyFilePath.ksy", "./outputDir", additionalFlags=[
45+
#you can expose additional params here
46+
])
47+
```
48+
49+
### Backends
50+
51+
#### CLI
52+
See https://github.com/kaitaiStructCompile/kaitaiStructCompileBackendCLI for more info.
53+
54+
#### Other backends
55+
More backends may be available in future. Even by third parties. Even for alternative Kaitai Struct compiler implementations.
56+
57+
58+
#### JVM backend
59+
I have created a JVM backend. It is **not distributed** and **YOU CANNOT OBTAIN IT** until [the issue with GPL license virality](https://github.com/kaitai-io/kaitai_struct/issues/466) is resolved.
60+
61+
Cons:
62+
* GPL virality
63+
* breaks sometimes
64+
65+
66+
Pros:
67+
* better security
68+
* works much faster
69+
70+
71+
### `importer`
72+
This is an importer allowing to import `ksy`s. Seamlessly compiles `ksy`s into python sources. Useful for playing in IPython shell.
73+
74+
75+
#### Usage
76+
77+
```python
78+
import kaitaiStructCompile.importer
79+
kaitaiStructCompile.importer._importer.searchDirs.append(Path("./dirWithKSYFiles")) # you can add a dir to search for KSY files.
80+
kaitaiStructCompile.importer._importer.flags["readStoresPos"] = True # you can set compiler flags, for more details see the JSON schema
81+
from kaitaiStructCompile.importer.test import Test
82+
Test # kaitaiStructCompile.importer.test.Test
83+
```
84+
85+
86+
### Build systems extensions
87+
This is a set of build system plugins allowing you to just specify `ksy` files you need to compile into python sources in process of building your package in a declarative way.
88+
89+
Supported build systems:
90+
* `setuptools`
91+
* `hatchling`
92+
* to enable also add an empty `[tool.hatch.build.hooks.kaitai_transpile]` section into your `pyproject.toml`.
93+
* `poetry`
94+
* works only with `poetry`, not `poetry-core` (because `poetry-core` has no support of plugins at all).
95+
* adds `kaitai transpile` command.
96+
97+
98+
#### Usage
99+
Just an add a property `tool.kaitai` into the `pyproject.toml`. This dict specified and documented with [the JSON Schema](./kaitaiStructCompile/schemas/config.schema.json), so read it carefully.
100+
101+
102+
Here a piece from one project with comments follows:
103+
```toml
104+
[build-system]
105+
requires = ["setuptools>=44", "wheel", "setuptools_scm[toml]>=3.4.3", "kaitaiStructCompile[toml]"] # we need this tool !
106+
...
107+
108+
# `tool.kaitai.repos` must contain 2-level nested dicts. The first component is usually repo URI, the second one is the refspec. But one can put there irrelevant things, if one needs for example compile multiple stuff from the same repo and refspec
109+
[tool.kaitai.repos."https://github.com/KOLANICH/kaitai_struct_formats.git"."mdt"]
110+
# one can put something irrelevant into the components of the section header, then he has to set `git` and `refspec`
111+
#git = "https://github.com/KOLANICH/kaitai_struct_formats.git"
112+
#refspec = "mdt"
113+
update = true # automatically download the freshest version of the repo each time the project is built
114+
search = true # glob all the spec from `inputDir`
115+
localPath = "kaitai_struct_formats" # Where (rel to `setup.py` dir) the repo of formats will be downloaded and from which location the compiler will use it.
116+
inputDir = "scientific/nt_mdt" # the directory we take KSYs from rel to `localPath`
117+
outputDir = "NTMDTRead/kaitai" # the directory we will put the generated file rel to setup.py dir
118+
119+
[tool.kaitai.repos."https://github.com/KOLANICH/kaitai_struct_formats.git"."mdt".formats]
120+
# here we declare our targets. MUST BE SET, even if empty!!!
121+
```
122+
123+
Or one can pass the dict of the similar structure to `setuptools.setup` directly using `kaitai` param (in this case you set all the paths yourself!!!), but it is disrecommended. Use the declarative config everywhere it is possible.
124+
125+
#### Real Examples
126+
127+
[1](https://github.com/KOLANICH-physics/NTMDTRead/blob/master/pyproject.toml)
128+
[2](https://github.com/KOLANICH-physics/SpecprParser.py/blob/master/pyproject.toml)
129+
[3](https://github.com/KOLANICH-tools/FrozenTable.py/blob/master/pyproject.toml)
130+
[4](https://github.com/KOLANICH-libs/lime.py/blob/master/pyproject.toml)
131+
[5](https://github.com/KOLANICH-libs/sunxi_fex.py/blob/master/pyproject.toml)
132+
[6](https://github.com/KOLANICH-tools/WindowsTelemetryViewer.py/blob/master/pyproject.toml)
133+
[7](https://github.com/KOLANICH-libs/MFCTool.py/blob/master/pyproject.toml)
134+
[8](https://github.com/KOLANICH-ML/RDataParser.py/blob/master/pyproject.toml)
135+
[9](https://github.com/KOLANICH-tools/inkwave.py/blob/master/pyproject.toml)
136+

‎UNLICENSE

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
This is free and unencumbered software released into the public domain.
2+
3+
Anyone is free to copy, modify, publish, use, compile, sell, or
4+
distribute this software, either in source code form or as a compiled
5+
binary, for any purpose, commercial or non-commercial, and by any
6+
means.
7+
8+
In jurisdictions that recognize copyright laws, the author or authors
9+
of this software dedicate any and all copyright interest in the
10+
software to the public domain. We make this dedication for the benefit
11+
of the public at large and to the detriment of our heirs and
12+
successors. We intend this dedication to be an overt act of
13+
relinquishment in perpetuity of all present and future rights to this
14+
software under copyright law.
15+
16+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19+
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20+
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21+
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22+
OTHER DEALINGS IN THE SOFTWARE.
23+
24+
For more information, please refer to <https://unlicense.org/>

0 commit comments

Comments
 (0)