Skip to content

Commit 524e4c3

Browse files
ramkrishna2910ramkrishna2910
committed
initial commit
0 parents  commit 524e4c3

File tree

1,283 files changed

+43136
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,283 files changed

+43136
-0
lines changed

‎.gitattributes

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
*.onnx filter=lfs diff=lfs merge=lfs -text
2+
*.pb filter=lfs diff=lfs merge=lfs -text
3+
*.zip filter=lfs diff=lfs merge=lfs -text
4+
*.tar filter=lfs diff=lfs merge=lfs -text
5+
*.gz filter=lfs diff=lfs merge=lfs -text
6+
*.pkl filter=lfs diff=lfs merge=lfs -text
7+
*.tar.gz filter=lfs diff=lfs merge=lfs -text
8+
*.tgz filter=lfs diff=lfs merge=lfs -text
9+
*.weight filter=lfs diff=lfs merge=lfs -text
10+
*.bias filter=lfs diff=lfs merge=lfs -text

‎.github/workflows/publish-to-test-pypi.yml

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI
2+
3+
on:
4+
push:
5+
branches: ["main", "canary"]
6+
tags:
7+
- v*
8+
pull_request:
9+
branches: ["main", "canary"]
10+
11+
jobs:
12+
build-n-publish:
13+
name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
14+
runs-on: ubuntu-latest
15+
steps:
16+
- uses: actions/checkout@main
17+
- uses: conda-incubator/setup-miniconda@v2
18+
with:
19+
miniconda-version: "latest"
20+
activate-environment: tkml
21+
python-version: ${{ matrix.python-version }}
22+
- name: Install pypa/build
23+
run: >-
24+
python -m pip install build --user
25+
- name: Build a binary wheel and a source tarball
26+
run: >-
27+
python -m build --sdist --wheel --outdir dist/ toolchain
28+
- name: Test wheel
29+
shell: bash -el {0}
30+
run: |
31+
python -m pip install --upgrade pip
32+
pip install dist/*.whl
33+
models=$(turnkey models location --quiet)
34+
turnkey $models/selftest/linear.py

‎.github/workflows/test_azure.yml

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# This workflow will install Python dependencies and run a basic test to ensure
2+
# that the Cloud/Azure CLI is still working
3+
name: Test Azure CLI
4+
5+
# TODO: enable these tests again once the repository is in the public domain
6+
7+
# on:
8+
# push:
9+
# branches: ["main"]
10+
# paths:
11+
# - toolchain/src/turnkeyml/cli/scale/**
12+
# pull_request:
13+
# branches: ["main"]
14+
# paths:
15+
# - toolchain/src/turnkeyml/cli/scale/**
16+
17+
# In the meantime, allow manual execution
18+
on:
19+
# Allows users to manually trigger the workflow using the GitHub UI
20+
workflow_dispatch:
21+
22+
permissions:
23+
contents: read
24+
25+
jobs:
26+
build_and_test:
27+
timeout-minutes: 30
28+
runs-on: ubuntu-latest
29+
steps:
30+
- uses: actions/checkout@v3
31+
- uses: conda-incubator/setup-miniconda@v2
32+
with:
33+
miniconda-version: "latest"
34+
activate-environment: tkml
35+
python-version: "3.8"
36+
- name: Install dependencies
37+
shell: bash -el {0}
38+
run: |
39+
sudo apt-get update
40+
sudo apt-get install -y azure-cli
41+
python -m pip install --upgrade pip
42+
pip install -e .
43+
pip install -r toolchain/src/turnkeyml/cli/cloud/requirements.txt
44+
- name: Run tests
45+
env:
46+
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
47+
TURNKEY_AZURE_PUBLIC_KEY: ${{ secrets.AZURE_SSH_PUBLIC_KEY }}
48+
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
49+
TURNKEY_AZURE_PASSWORD: ${{ secrets.TURNKEY_AZURE_PASSWORD }}
50+
shell: bash -el {0}
51+
run: |
52+
az login --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_PWD }}
53+
54+
ssh-agent -a $SSH_AUTH_SOCK > /dev/null
55+
ssh-add - <<< "${{ secrets.AZURE_SSH_PRIVATE_KEY }}"
56+
57+
mkdir ~/.ssh
58+
touch ~/.ssh/known_hosts
59+
60+
cd src/turnkeyml/cli/cloud
61+
62+
python control.py create setup selftest info stop --name github-test-azure-single --hardware cpu-small
63+
64+
python control.py create setup selftest info stop --cluster --name github-test-azure-cluster --size 2 --hardware cpu-small
65+
python control.py start run --cluster --name github-test-azure-cluster --size 2 --hardware cpu-small --input-files onnxmodelzoo/toolchain/models/selftest/*.py
66+
clean_up:
67+
needs: build_and_test
68+
if: always()
69+
runs-on: ubuntu-latest
70+
steps:
71+
- uses: actions/checkout@v3
72+
- name: Install az cli
73+
run: |
74+
sudo apt-get update
75+
sudo apt-get install -y azure-cli
76+
- name: Delete resource group
77+
run: |
78+
az login --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_PWD }}
79+
az group delete --name github-test-azure-single-rg --yes
80+
az group delete --name github-test-azure-cluster-rg --yes

‎.github/workflows/test_build_api.yml

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# This workflow will install Python dependencies, run tests and lint with a single version of Python
2+
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
3+
4+
name: Lint and Test Build API
5+
6+
on:
7+
push:
8+
branches: ["main", "canary"]
9+
pull_request:
10+
branches: ["main", "canary"]
11+
12+
permissions:
13+
contents: read
14+
15+
jobs:
16+
build-the-build-api:
17+
env:
18+
TURNKEY_DEBUG: True
19+
TURNKEY_TRACEBACK: True
20+
strategy:
21+
matrix:
22+
python-version: ["3.8", "3.10"]
23+
os: [ubuntu-latest, windows-latest]
24+
runs-on: ${{ matrix.os }}
25+
steps:
26+
- uses: actions/checkout@v2
27+
- name: Set up Miniconda with 64-bit Python
28+
uses: conda-incubator/setup-miniconda@v2
29+
with:
30+
miniconda-version: "latest"
31+
activate-environment: tkml
32+
python-version: ${{ matrix.python-version }}
33+
architecture: "x64"
34+
- name: Install dependencies
35+
shell: bash -el {0}
36+
run: |
37+
python -m pip install --upgrade pip
38+
conda install pylint
39+
pip install -e toolchain[tensorflow] --no-cache-dir
40+
pip install transformers
41+
python -m pip check
42+
- name: Lint with PyLint
43+
shell: bash -el {0}
44+
run: |
45+
pylint toolchain/src/turnkeyml/build --rcfile toolchain/.pylintrc
46+
pylint toolchain/examples/build_api --rcfile toolchain/.pylintrc
47+
- name: Test with unittest
48+
shell: bash -el {0}
49+
run: |
50+
51+
# build api examples
52+
# Note: we clear the default cache location prior to each example run
53+
rm -rf ~/.cache/turnkey_test_cache
54+
python toolchain/examples/build_api/hello_torch_world.py
55+
rm -rf ~/.cache/turnkey_test_cache
56+
python toolchain/examples/build_api/hello_keras_world.py
57+
rm -rf ~/.cache/turnkey_test_cache
58+
python toolchain/examples/build_api/hello_onnx_world.py
59+
rm -rf ~/.cache/turnkey_test_cache
60+
python toolchain/examples/build_api/randomforest.py
61+
rm -rf ~/.cache/turnkey_test_cache
62+
python toolchain/examples/build_api/xgbclassifier.py
63+
rm -rf ~/.cache/turnkey_test_cache
64+
python toolchain/examples/build_api/lgbmclassifier.py
65+
rm -rf ~/.cache/turnkey_test_cache
66+
python toolchain/examples/build_api/kneighborsclassifier.py
67+
rm -rf ~/.cache/turnkey_test_cache
68+
python toolchain/examples/build_api/build_name.py
69+
rm -rf ~/.cache/turnkey_test_cache
70+
python toolchain/examples/build_api/cache_dir.py
71+
rm -rf ~/.cache/turnkey_test_cache
72+
python toolchain/examples/build_api/no_monitor.py
73+
rm -rf ~/.cache/turnkey_test_cache
74+
python toolchain/examples/build_api/rebuild_always.py
75+
rm -rf ~/.cache/turnkey_test_cache
76+
python toolchain/examples/build_api/rebuild_never.py
77+
rm -rf ~/.cache/turnkey_test_cache
78+
python toolchain/examples/build_api/sequence.py
79+
80+
# build api tests
81+
python toolchain/test/build_model.py

‎.github/workflows/test_gpu_turnkey.yml

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# This workflow will install Python dependencies, run tests and lint with a single version of Python
2+
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
3+
4+
name: Test TurnkeyML on Azure GPUs
5+
6+
on:
7+
# Allows users to manually trigger the workflow using the GitHub UI
8+
workflow_dispatch:
9+
10+
permissions:
11+
contents: read
12+
13+
jobs:
14+
start_vm:
15+
timeout-minutes: 10
16+
runs-on: ubuntu-latest
17+
steps:
18+
- uses: actions/checkout@v3
19+
- name: Install az cli
20+
run: |
21+
sudo apt-get update
22+
sudo apt-get install -y azure-cli
23+
- name: Turn on Azure VM
24+
run: |
25+
az login --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_PWD }}
26+
az vm wait --name morecuda --resource-group morecuda_group --custom "instanceView.statuses[?code=='PowerState/deallocated']" --interval 30
27+
az vm start --name morecuda --resource-group morecuda_group
28+
az vm wait --created --name morecuda --resource-group morecuda_group
29+
az vm run-command invoke --name morecuda --resource-group morecuda_group --command-id RunShellScript --scripts 'export RUNNER_ALLOW_RUNASROOT="1" && export HOME="/home/azureuser" && bash /home/azureuser/actions-runner/run.sh &'
30+
build_and_test:
31+
needs: start_vm
32+
timeout-minutes: 10
33+
if: ${{ needs.start_vm.result == 'success' }}
34+
runs-on: self-hosted
35+
env:
36+
TURNKEY_DEBUG: True
37+
TURNKEY_TRACEBACK: True
38+
strategy:
39+
matrix:
40+
python-version: ["3.8"]
41+
steps:
42+
- uses: actions/checkout@v3
43+
- uses: conda-incubator/setup-miniconda@v2
44+
with:
45+
miniconda-version: "latest"
46+
activate-environment: tkml
47+
python-version: ${{ matrix.python-version }}
48+
- name: Install Docker
49+
shell: bash -el {0}
50+
run: |
51+
sudo mkdir -p /etc/apt/keyrings
52+
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
53+
sudo chmod a+r /etc/apt/keyrings/docker.gpg
54+
echo \
55+
"deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
56+
"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
57+
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
58+
sudo apt-get update
59+
sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
60+
sudo chmod 666 /var/run/docker.sock
61+
- name: Install dependencies
62+
shell: bash -el {0}
63+
run: |
64+
python -m pip install --upgrade pip
65+
conda install pylint
66+
pip install -e toolchain
67+
pip install transformers
68+
python -m pip check
69+
- name: Test with unittest
70+
shell: bash -el {0}
71+
run: |
72+
# E2E tests
73+
python toolchain/test/gpu.py
74+
deallocate_vm:
75+
needs: build_and_test
76+
if: always()
77+
runs-on: ubuntu-latest
78+
steps:
79+
- uses: actions/checkout@v3
80+
- name: Install az cli
81+
run: |
82+
sudo apt-get update
83+
sudo apt-get install -y azure-cli
84+
- name: Deallocate Azure VM
85+
run: |
86+
az login --username ${{ secrets.AZURE_CLIENT_ID }} --password ${{ secrets.AZURE_CLIENT_PWD }}
87+
az vm deallocate --name morecuda --resource-group morecuda_group

0 commit comments

Comments
 (0)