-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathjustfile
executable file
·172 lines (157 loc) · 7.69 KB
/
justfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
#!/usr/bin/env just --justfile
# Source the .env file
set dotenv-load := true
set export
# List recipes
list:
@just --list --unsorted
# Setup the environment
install:
conda env create -f environment.yml
python -m pip install --upgrade pip
# Update the environment. Run this command after checking out any code changes
update:
conda env update --file environment.yml --prune
# Setup the database
setup-db:
#!/usr/bin/env bash
export PYTHONPATH=.
rm -rf tator
git clone --recurse-submodules https://github.com/mbari-org/tator
cd tator && git checkout 1.2.5
cp example-env .env && make tator && make superuser
# TODO: Add a command to initialize the database from the yaml file
# See sightwire code for an example of how to do this
## Initialize the database
#init-db:
# #!/usr/bin/env bash
# export PYTHONPATH=.
# export PATH="$PATH:~/miniconda3/bin/"
# conda run -n mbari_aidata --no-capture-output python3 mbari_aidata db create
# Stop the docker development environment
stop-docker-dev:
#!/usr/bin/env bash
docker stop nginx_images
docker stop redis-test
# cd tator && make down
# Setup the docker development environment
setup-docker-dev:
#!/usr/bin/env bash
export PATH=$PATH:/usr/local/bin
docker stop nginx_images
docker rm nginx_images
docker run -d -p 8082:8082 \
-v $PWD/tests/data/:/data \
-v $PWD/tests/nginx.conf:/etc/nginx/conf.d/default.conf \
--restart always \
--name nginx_images nginx:1.23.3
# Get the IP address of the host and add it to the host: field in all the test yaml files
export HOST_IP=$(ipconfig getifaddr en0)
git checkout tests/config/*.yml
sed -i '' "s/host: localhost/host: $HOST_IP/g" tests/config/*.yml
docker volume create redis-test
docker stop redis-test && docker rm redis-test || true
docker run \
--name redis-test \
--env-file .env \
-p 6379:6379 \
-v redis-test:/var/lib/redis-stack \
--restart always \
redis/redis-stack-server \
/bin/sh -c 'redis-stack-server --port 6382 --appendonly yes --appendfsync everysec --requirepass "${REDIS_PASSWD:?REDIS_PASSWD variable is not set}"'
# Build the docker images for all platforms
build-docker:
#!/usr/bin/env bash
export PATH=$PATH:/usr/local/bin
# Get the release version from the git tag and strip the v from the version
export RELEASE_VERSION=$(git describe --tags --abbrev=0)
export RELEASE_VERSION=${RELEASE_VERSION:1}
echo "Building docker images for release version: $RELEASE_VERSION"
docker buildx create --name mybuilder --platform linux/amd64,linux/arm64 --use
docker buildx build --push --platform linux/amd64,linux/arm64 -t mbari/mbari_aidata:$RELEASE_VERSION --build-arg IMAGE_URI=mbari/mbari_aidata:$RELEASE_VERSION -f docker/Dockerfile .
docker buildx build --push --platform linux/amd64 -t mbari/mbari_aidata:$RELEASE_VERSION-cuda124 --build-arg IMAGE_URI=mbari/mbari_aidata:$RELEASE_VERSION-cuda124 -f docker/Dockerfile.cuda .
docker push mbari/mbari_aidata:$RELEASE_VERSION
# Install development dependencies. Run before running tests
install-dev:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
conda run -n mbari_aidata --no-capture-output python3 -m pip install -r requirements-dev.txt
# Load i2map images
load-i2map:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata load images \
--config ./tests/config/config_i2map.yml \
--input ./tests/data/i2map --token $TATOR_TOKEN
# Test loading of i2map images
test-load-i2map:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output pytest -r tests/test_load_media.py -k test_load_image_i2map
# Test loading of i2map images
test-load-cfe:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output pytest -r tests/test_load_media.py -k test_load_image_cfe
# Test dry-run loading of images or videos
test-dryrun:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output pytest -r tests/test_load_media.py -k test_load_image_dryrun
time conda run -n mbari_aidata --no-capture-output pytest -r tests/test_load_media.py -k test_load_video_dryrun
# Download all verified data from the i2map project at 300m depth
download-300m-data:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata download dataset --base-path ./data/i2map --version Baseline --depth 300 --labels "all" --config ./mbari_aidata/config/config_i2map.yml
# Download all verified data from the i2map project at 300m depth with a minimum score of 0.97
download-300m-data-gtp97:
#!/usr/bin/env bash
export PYTHONPATH=.
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata download dataset --base-path ./data/i2map --version Baseline --depth 300 --min-score 0.97 --labels "all" --config ./mbari_aidata/config/config_i2map.yml
# Download the 300m data for the Atolla dataset with the mega-vits-track-gcam version
download-atolla-data:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata download dataset --voc --version mega-vits-track-gcam --labels "Atolla" --crop-roi --config ./mbari_aidata/config/config_bio.yml
# Download all verified data from the bio project and collapse the labels to a single class
download-single-class-data:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata download dataset --config --single-class "marineorganism" --version Baseline --labels "Atolla,Gymnopraia lapislazula" --voc --config ./mbari_aidata/config/config_bio.yml
# Download the pinniped data from the UAV project
download-pinniped-data:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata download dataset --labels "Pinniped" --crop-roi --config ./mbari_aidata/config/config_uav.yml
download-jelly-data:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata download dataset --crop-roi --labels "Jelly" --config ./mbari_aidata/config/config_uav.yml
download-bird-data:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata download dataset --crop-roi --verified --labels "Bird" --config ./mbari_aidata/config/config_uav.yml
# Download the copepod data from the CFE project
download-copepod-data:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata download dataset --version Baseline --labels "copepod" --config ./mbari_aidata/config/config_cfe.yml
# Download the 25000_depth_v1 section data from the CFE project
download-section-data:
#!/usr/bin/env bash
export PYTHONPATH=.
export PATH="$PATH:$CONDA_PREFIX/bin"
time conda run -n mbari_aidata --no-capture-output python3 mbari_aidata download dataset --base-path ./data/i2map --version Baseline --section "25000_depth_v1" --labels "all" --config ./mbari_aidata/config/config_cfe.yml