Skip to content

Commit 5ae869d

Browse files
author
gzheng2
committed
initial checkin
1 parent 0cd98af commit 5ae869d

21 files changed

+911
-0
lines changed

‎app/analysis-table-app/.dockerignore

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
secrets.json
2+
.agave
3+
.agpy
4+
*.md
5+
env/*
6+
*.pyc
7+
.DS_Store
8+
.git
9+
.env
10+
.aws
11+
ida_rsa.*
12+
*.key
13+
*.pem
14+
*.crt
15+
*.pkcs8
16+
.docker

‎app/analysis-table-app/.gitignore

+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
# Abaco and Reactors-specific
2+
.agave
3+
.agpy
4+
secrets.json
5+
.ACTOR_ID*
6+
deploy-*
7+
8+
# Byte-compiled / optimized / DLL files
9+
__pycache__/
10+
*.py[cod]
11+
*$py.class
12+
13+
# C extensions
14+
*.so
15+
16+
# Distribution / packaging
17+
.Python
18+
build/
19+
develop-eggs/
20+
dist/
21+
downloads/
22+
eggs/
23+
.eggs/
24+
lib/
25+
lib64/
26+
parts/
27+
sdist/
28+
var/
29+
wheels/
30+
*.egg-info/
31+
.installed.cfg
32+
*.egg
33+
MANIFEST
34+
35+
# PyInstaller
36+
# Usually these files are written by a python script from a template
37+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
38+
*.manifest
39+
*.spec
40+
41+
# Installer logs
42+
pip-log.txt
43+
pip-delete-this-directory.txt
44+
45+
# Unit test / coverage reports
46+
htmlcov/
47+
.tox/
48+
.coverage
49+
.coverage.*
50+
.cache
51+
nosetests.xml
52+
coverage.xml
53+
*.cover
54+
.hypothesis/
55+
56+
# Translations
57+
*.mo
58+
*.pot
59+
60+
# Django stuff:
61+
*.log
62+
.static_storage/
63+
.media/
64+
local_settings.py
65+
66+
# Flask stuff:
67+
instance/
68+
.webassets-cache
69+
70+
# Scrapy stuff:
71+
.scrapy
72+
73+
# Sphinx documentation
74+
docs/_build/
75+
76+
# PyBuilder
77+
target/
78+
79+
# Jupyter Notebook
80+
.ipynb_checkpoints
81+
82+
# pyenv
83+
.python-version
84+
85+
# celery beat schedule file
86+
celerybeat-schedule
87+
88+
# SageMath parsed files
89+
*.sage.py
90+
91+
# Environments
92+
.env
93+
.venv
94+
env/
95+
venv/
96+
ENV/
97+
env.bak/
98+
venv.bak/
99+
100+
# Spyder project settings
101+
.spyderproject
102+
.spyproject
103+
104+
# Rope project settings
105+
.ropeproject
106+
107+
# mkdocs documentation
108+
/site
109+
110+
# mypy
111+
.mypy_cache/

‎app/analysis-table-app/Dockerfile

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM sd2e/base:ubuntu18
2+
3+
RUN apt-get update && \
4+
apt-get install -y \
5+
build-essential \
6+
python3 \
7+
python3-dev \
8+
python3-pip && \
9+
apt-get clean
10+
11+
RUN pip3 install --upgrade pip
12+
13+
ADD src /src
14+
ADD config.yml /config.yml

‎app/analysis-table-app/analysis-table-app-0.1.0/_util/VERSION

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
0.11.7

‎app/analysis-table-app/analysis-table-app-0.1.0/_util/container_exec.sh

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#!/usr/bin/env bash
2+
3+
# Fetch the container runtime script on the fly, allowing us to update for
4+
# apps as we improve it
5+
# curl -skL -O https://raw.githubusercontent.com/sd2e/sd2e-cli/master/install/runtime.sh
6+
7+
container_exec() {
8+
9+
# [TODO] Check for existence of docker or singularity executable
10+
# [TODO] Enable honoring a DEBUG global
11+
# [TODO] Figure out how to accept more optional arguments (env-file, etc)
12+
# [TODO] Better error handling and reporting
13+
# [TODO] Handle "urllib2.URLError: <urlopen error [Errno -3] Temporary failure in name resolution>"
14+
15+
local CONTAINER_IMAGE=$1
16+
shift
17+
local COMMAND=$1
18+
shift
19+
local PARAMS=$@
20+
21+
# A litte logging to help with the edge cases
22+
if [ ! -z "$DEBUG" ];
23+
then
24+
local _PID=$$
25+
echo $CONTAINER_IMAGE > .container_exec.${_PID}.log
26+
echo $COMMAND >> .container_exec.${_PID}.log
27+
echo $PARAMS >> .container_exec.${_PID}.log
28+
echo $PWD >> .container_exec.${_PID}.log
29+
echo $(ls $PWD) >> .container_exec.${_PID}.log
30+
env > .container_exec.${_PID}.env
31+
fi
32+
33+
# Detect container engine
34+
local _CONTAINER_APP=$(which singularity)
35+
36+
echo $_CONTAINER_APP
37+
38+
if [ ! -z "${_CONTAINER_APP}" ]
39+
then
40+
_CONTAINER_ENGINE="singularity"
41+
else
42+
_CONTAINER_APP=$(which docker)
43+
if [ ! -z "${_CONTAINER_APP}" ]
44+
then
45+
echo "setting to docker"
46+
_CONTAINER_ENGINE="docker"
47+
fi
48+
fi
49+
50+
if [ -z "$SINGULARITY_PULLFOLDER" ];
51+
then
52+
if [ ! -z "$STOCKYARD" ];
53+
then
54+
SINGULARITY_PULLFOLDER="${STOCKYARD}/.singularity"
55+
else
56+
SINGULARITY_PULLFOLDER="$HOME/.singularity"
57+
fi
58+
fi
59+
60+
if [ -z "$SINGULARITY_CACHEDIR" ];
61+
then
62+
if [ ! -z "$STOCKYARD" ];
63+
then
64+
SINGULARITY_CACHEDIR="${STOCKYARD}/.singularity"
65+
else
66+
SINGULARITY_CACHEDIR="$HOME/.singularity"
67+
fi
68+
fi
69+
70+
local _UID=$(id -u)
71+
local _GID=$(id -g)
72+
chmod -R g+rwxs .
73+
umask 002 .
74+
75+
76+
if [[ "$_CONTAINER_ENGINE" == "docker" ]]; then
77+
echo "hey, it's docker"
78+
#local OPTS="--network=none --cpus=1.0000 --memory=1G --device-read-iops=/dev/sda:1500 --device-read-iops=/dev/sda:1500"
79+
80+
# Set group ownership on all files making them readable by archive process
81+
OPTS="$OPTS --rm --user=0:${_GID} -v $PWD:/data:rw -w /data"
82+
if [ ! -z "$ENVFILE" ]
83+
then
84+
OPTS="$OPTS --env-file ${ENVFILE}"
85+
fi
86+
if [ "$INTERACTIVE" == "1" ]; then
87+
OPTS="-it $OPTS"
88+
fi
89+
if [ ! -z "$DEBUG" ];
90+
then
91+
set -x
92+
fi
93+
if [ "$NO_PULL" != "1" ]; then
94+
echo "Pulling ${CONTAINER_IMAGE}. Set NO_PULL=1 to prevent this."
95+
docker pull ${CONTAINER_IMAGE} >/dev/null 2>&1 || true
96+
fi
97+
98+
echo "docker run OPTS CONTAINER_IMAGE COMMAND PARAMS:" $OPTS ${CONTAINER_IMAGE} ${COMMAND} ${PARAMS}
99+
docker run $OPTS ${CONTAINER_IMAGE} ${COMMAND} ${PARAMS}
100+
STAT="$?"
101+
# docker run $OPTS ${CONTAINER_IMAGE} ${COMMAND} ${PARAMS} &&
102+
docker run $OPTS bash chmod -R a+rw .
103+
if [ ! -z "$DEBUG" ];
104+
then
105+
set +x
106+
fi
107+
return $STAT
108+
elif [[ "$_CONTAINER_ENGINE" == "singularity" ]];
109+
then
110+
# [TODO] Detect and deal if an .img has been passed it (rare)
111+
echo "singularity exec:" ${CONTAINER_IMAGE} ${COMMAND} ${PARAMS}
112+
singularity exec docker://${CONTAINER_IMAGE} ${COMMAND} ${PARAMS}
113+
STAT="$?"
114+
return $STAT
115+
else
116+
echo "_CONTAINER_ENGINE needs to be 'docker' or 'singularity' [$_CONTAINER_ENGINE]"
117+
fi
118+
119+
}
120+
121+
122+
count_logical_cores() {
123+
124+
local _count_cores=4
125+
local _uname=$(uname)
126+
127+
if [ "$_uname" == "Darwin" ]
128+
then
129+
_count_cores=$(sysctl -n hw.logicalcpu)
130+
elif [ "$_uname" == "Linux" ]
131+
then
132+
_count_cores=$(grep -c processor /proc/cpuinfo)
133+
fi
134+
135+
echo $_count_cores
136+
137+
}
138+
139+
auto_maxthreads() {
140+
141+
local hwcore=$(count_logical_cores)
142+
hwcore=$((hwcore-1))
143+
echo $hwcore
144+
145+
}

‎app/analysis-table-app/analysis-table-app-0.1.0/app.json.j2

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{
2+
"name": "{{ app.name }}",
3+
"version": "{{ app.version }}",
4+
"executionType": "HPC",
5+
"executionSystem": "hpc-tacc-wrangler",
6+
"parallelism": "SERIAL",
7+
"deploymentPath": "apps/{{ app.bundle }}",
8+
"deploymentSystem": "data-sd2e-app-assets",
9+
"label": "analysis table",
10+
"modules": [
11+
"load tacc-singularity"
12+
],
13+
"shortDescription": "First level analysis post data-converge",
14+
"templatePath": "runner-template.sh",
15+
"testPath": "tester.sh",
16+
"inputs": [],
17+
"parameters": [
18+
{
19+
"id": "CONTAINER_IMAGE",
20+
"value": {
21+
"default": "{{ docker.username }}/{{ docker.repo }}:{{ docker.tag }}",
22+
"description": "Container Image. Do not edit.",
23+
"type": "string",
24+
"visible": false,
25+
"required": true
26+
}
27+
},
28+
{
29+
"id": "experiment_id",
30+
"value": {
31+
"type": "string",
32+
"visible": true,
33+
"required": true
34+
}
35+
}
36+
],
37+
"outputs": [
38+
{
39+
"id": "output",
40+
"value": {
41+
"default": "samples_labelled.csv"
42+
},
43+
"semantics": {
44+
"ontology": [
45+
"http://edamontology.org/format_3752"
46+
]
47+
},
48+
"details": {
49+
"label": "Contains dataframe from the analysis table code"
50+
}
51+
}
52+
]
53+
}

‎app/analysis-table-app/analysis-table-app-0.1.0/job.json.j2

Whitespace-only changes.

‎app/analysis-table-app/analysis-table-app-0.1.0/runner-template.sh

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Allow over-ride
2+
if [ -z "${CONTAINER_IMAGE}" ]
3+
then
4+
version=$(cat ./_util/VERSION)
5+
CONTAINER_IMAGE="index.docker.io/sd2e/sample-qc-app:$version"
6+
fi
7+
. _util/container_exec.sh
8+
9+
log(){
10+
mesg "INFO" $@
11+
}
12+
13+
die() {
14+
mesg "ERROR" $@
15+
${AGAVE_JOB_CALLBACK_FAILURE}
16+
exit 0
17+
}
18+
19+
mesg() {
20+
lvl=$1
21+
shift
22+
message=$@
23+
echo "[$lvl] $(utc_date) - $message"
24+
}
25+
26+
utc_date() {
27+
echo $(date -u +"%Y-%m-%dT%H:%M:%SZ")
28+
}
29+
30+
#### BEGIN SCRIPT LOGIC
31+
echo "invoking container_exec" ${CONTAINER_IMAGE} ${experiment_id}
32+
container_exec ${CONTAINER_IMAGE} python3 /src/analysis.py --experiment-id ${experiment_id}

‎app/analysis-table-app/analysis-table-app-0.1.0/tester.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/usr/bin/env bash
2+
3+
return 0

‎app/analysis-table-app/app.ini

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
; App details
2+
[app]
3+
;appalias = analysis-table-app
4+
name = analysis-table-app
5+
version = 0.1.0
6+
bundle = analysis-table-app-0.1.0
7+
8+
[docker]
9+
***REMOVED***
10+
repo = analysis-table
11+
tag = 0.1.0
12+
dockerfile = Dockerfile

0 commit comments

Comments
 (0)