diff --git a/Dockerfile_SnakemakePipeline b/Dockerfile_SnakemakePipeline deleted file mode 100644 index d45fe59..0000000 --- a/Dockerfile_SnakemakePipeline +++ /dev/null @@ -1,26 +0,0 @@ -FROM continuumio/miniconda - -COPY pipelines/pipeline_controller/requirements.txt . -COPY pipelines/pipeline_controller/installer_Rpackage.R . - -RUN conda create -n pipeline_controller_base python=3.8.2 R=3.6 -SHELL ["conda", "run", "-n", "pipeline_controller_base", "/bin/bash", "-c"] - -RUN pip install -r requirements.txt -RUN Rscript installer_Rpackage.R - -RUN python -c "import flask" - -#FROM snakemake/snakemake - -#COPY pipelines/pipeline_controller/installer_Rpackage.R . -#COPY pipelines/pipeline_controller/requirements.txt . - -# Should solve calling all requirements in saperate pipeline folders -#RUN apt-get update && apt-get install -y r-base -#RUN Rscript ./installer_Rpackage.R -#RUN pip install -r ./requirements.txt - -#WORKDIR /pipelines/pipeline_controller/ -#ENTRYPOINT ["gunicorn", "--bind", "0.0.0.0:5000", \ -#"--workers", "2", "--threads", "4", "--worker-class", "gthread", "connector:app"] \ No newline at end of file diff --git a/Dockerfile_jupyterNotebook b/Dockerfile_jupyterNotebook deleted file mode 100644 index d073493..0000000 --- a/Dockerfile_jupyterNotebook +++ /dev/null @@ -1,7 +0,0 @@ -FROM jupyter/datascience-notebook - -COPY notebook/installers/installer_Rpackage.R /installer_Rpackage.R -COPY notebook/installers/requirements.txt /requirements.txt - -RUN Rscript /installer_Rpackage.R -RUN pip install -r /requirements.txt \ No newline at end of file diff --git a/README.md b/README.md index 29d5a44..8ff822f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ * Current members: Kicheol Kim, Junhee Yoon * Please, leave a message in **Discussions** tab if you have any question and requests * Please use docker image to analyze the data. AWS module is ready and Please ask to members for getting auth f AWS is needed to analze data. -* Our data is living in S3 bucket +* Our data is located in S3 bucket ### Goal * Finding potential biomarkers and therapeutic target for helping multiple sclerosis patients, **reference**: [Cell type-specific transcriptomics identifies neddylation as a novel therapeutic target in multiple sclerosis](https://pubmed.ncbi.nlm.nih.gov/33374005/) @@ -21,13 +21,15 @@ * https://openkbc.github.io/multiple_sclerosis_proj/ ### Usage of docker container -* 2 images are composing up for jupyter notebook and workflow. The workflow image does not have controller currently, so user needs to get inside to control it by using docker attach. +* 4 images are needed to use services (notebook, pipelines, celery and redis) +* We are using docker registry to distribute images, please refer to [here](https://hub.docker.com/repository/docker/swiri021/openkbc_msproject/general) ![overview](README_resource/overview_recent.png) * Containers ```shell -docker-compose -f docker-compose.yaml up --build # composing up +docker-compose -f docker-compose.yaml up --build # composing up by the codes or +docker-compose -f docker-compose.example.yaml up # composing up by using the registry ``` * Jupyter notebook diff --git a/aws_deployment/docker-compose.yaml b/aws_deployment/docker-compose.yaml deleted file mode 100644 index e0cd7f7..0000000 --- a/aws_deployment/docker-compose.yaml +++ /dev/null @@ -1,57 +0,0 @@ -version: "3" -services: - notebook: # Notebook - build: - context: . - dockerfile: Dockerfile_jupyterNotebook - volumes: - - /home/ubuntu/MSProject/multiple_sclerosis_proj/notebook/notebook_lib:/home/jovyan/work/notebook_lib - - /home/ubuntu/MSProject/multiple_sclerosis_proj/notebook/notebook_utils:/home/jovyan/work/notebook_utils - - /home/ubuntu/MSProject/multiple_sclerosis_proj/notebook/notebook_archive:/home/jovyan/work/notebook_archive - - /home/ubuntu/MSProject/multiple_sclerosis_proj/notebook/resultFiles:/home/jovyan/work/resultFiles - - /home/ubuntu/MSProject/multiple_sclerosis_proj/data:/home/jovyan/MainData - ports: - - 8888:8888 - container_name: notebookContainer - - pipelines: # Pipelines - build: - context: . - dockerfile: Dockerfile_SnakemakePipeline - deploy: - resources: - limits: - memory: 4000m - volumes: - - /home/ubuntu/MSProject/multiple_sclerosis_proj/pipelines:/pipelines - - /home/ubuntu/MSProject/multiple_sclerosis_proj/data:/MainData - - /home/ubuntu/MSProject/multiple_sclerosis_proj/notebook/resultFiles:/Output - ports: - - 80:5000 - depends_on: - - redis - container_name: pipelineContainer - working_dir: /pipelines/pipeline_controller - command: conda run -n pipeline_controller_base gunicorn --bind 0.0.0.0:5000 --workers 2 --threads 4 --worker-class gthread connector:app - - redis: # redis - image: redis:alpine - command: redis-server - ports: - - 6379:6379 - container_name: redisServer - - celery: # celery - build: - context: . - dockerfile: Dockerfile_SnakemakePipeline - volumes: - - /home/ubuntu/MSProject/multiple_sclerosis_proj/pipelines:/pipelines - - /home/ubuntu/MSProject/multiple_sclerosis_proj/data:/MainData - - /home/ubuntu/MSProject/multiple_sclerosis_proj/resultFiles:/Output - working_dir: /pipelines/pipeline_controller/ - command: conda run -n pipeline_controller_base celery -A app.celery worker --loglevel=info - depends_on: - - redis - - pipelines - container_name: celeryContainer \ No newline at end of file diff --git a/docker-compose.AWS.yaml b/docker-compose.AWS.yaml index e0cd7f7..c15ac03 100644 --- a/docker-compose.AWS.yaml +++ b/docker-compose.AWS.yaml @@ -2,8 +2,7 @@ version: "3" services: notebook: # Notebook build: - context: . - dockerfile: Dockerfile_jupyterNotebook + context: ./notebook volumes: - /home/ubuntu/MSProject/multiple_sclerosis_proj/notebook/notebook_lib:/home/jovyan/work/notebook_lib - /home/ubuntu/MSProject/multiple_sclerosis_proj/notebook/notebook_utils:/home/jovyan/work/notebook_utils @@ -16,8 +15,7 @@ services: pipelines: # Pipelines build: - context: . - dockerfile: Dockerfile_SnakemakePipeline + context: ./pipelines deploy: resources: limits: @@ -43,8 +41,7 @@ services: celery: # celery build: - context: . - dockerfile: Dockerfile_SnakemakePipeline + context: ./pipelines volumes: - /home/ubuntu/MSProject/multiple_sclerosis_proj/pipelines:/pipelines - /home/ubuntu/MSProject/multiple_sclerosis_proj/data:/MainData diff --git a/docker-compose.example.yaml b/docker-compose.example.yaml new file mode 100644 index 0000000..0fd8412 --- /dev/null +++ b/docker-compose.example.yaml @@ -0,0 +1,54 @@ +version: "3" +services: + notebook: # Notebook + build: + image: swiri021/openkbc_msproject:notebookcontainer1 + volumes: + - your_library_path:/home/jovyan/work/notebook_lib # Anything you want to import, but you can set up our codes from the github + - your_utils_path:/home/jovyan/work/notebook_utils + - your_archive_path:/home/jovyan/work/notebook_archive + - your_resultFile_path:/home/jovyan/work/resultFiles + - s3data_path_in_your_local:/home/jovyan/MainData # S3 data from our bucket + ports: + - 8888:8888 + container_name: notebookContainer + + pipelines: # Pipelines + build: + image: swiri021/openkbc_msproject:pipelinecontainer1 + deploy: + resources: + limits: + memory: 4000m + volumes: + - pipelines_code_path:/pipelines # This codes from our github(pipeline folder) + - s3data_path_in_your_local:/MainData + - your_resultFile_path:/Output # Directly connected to notebook + ports: + - 80:5000 + depends_on: + - redis + container_name: pipelineContainer + working_dir: /pipelines/pipeline_controller + command: conda run -n pipeline_controller_base gunicorn --bind 0.0.0.0:5000 --workers 2 --threads 4 --worker-class gthread connector:app + + redis: # redis + image: redis:alpine + command: redis-server + ports: + - 6379:6379 + container_name: redisServer + + celery: # celery + build: + image: swiri021/openkbc_msproject:celerycontainer1 + volumes: # Celery volume path should be the same with pipeline volume + - pipelines_code_path:/pipelines + - s3data_path_in_your_local:/MainData + - your_resultFile_path:/Output + working_dir: /pipelines/pipeline_controller/ + command: conda run -n pipeline_controller_base celery -A app.celery worker --loglevel=info + depends_on: + - redis + - pipelines + container_name: celeryContainer \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index fc64471..6600719 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -2,8 +2,8 @@ version: "3" services: notebook: # Notebook build: - context: . - dockerfile: Dockerfile_jupyterNotebook + context: ./notebook + #image: swiri021/openkbc_msproject:notebookcontainer1 volumes: - /Users/junheeyun/OpenKBC/multiple_sclerosis_proj/notebook/notebook_lib:/home/jovyan/work/notebook_lib - /Users/junheeyun/OpenKBC/multiple_sclerosis_proj/notebook/notebook_utils:/home/jovyan/work/notebook_utils @@ -16,8 +16,8 @@ services: pipelines: # Pipelines build: - context: . - dockerfile: Dockerfile_SnakemakePipeline + context: ./pipelines + #image: swiri021/openkbc_msproject:pipelinecontainer1 deploy: resources: limits: @@ -43,8 +43,8 @@ services: celery: # celery build: - context: . - dockerfile: Dockerfile_SnakemakePipeline + context: ./pipelines + #image: swiri021/openkbc_msproject:celerycontainer1 volumes: - /Users/junheeyun/OpenKBC/multiple_sclerosis_proj/pipelines:/pipelines - /Users/junheeyun/OpenKBC/multiple_sclerosis_proj/data:/MainData diff --git a/notebook/Dockerfile b/notebook/Dockerfile new file mode 100644 index 0000000..ab674b0 --- /dev/null +++ b/notebook/Dockerfile @@ -0,0 +1,7 @@ +FROM jupyter/datascience-notebook + +COPY installers/installer_Rpackage.R /installer_Rpackage.R +COPY installers/requirements.txt /requirements.txt + +RUN Rscript /installer_Rpackage.R +RUN pip install -r /requirements.txt \ No newline at end of file diff --git a/pipelines/Dockerfile b/pipelines/Dockerfile new file mode 100644 index 0000000..622169b --- /dev/null +++ b/pipelines/Dockerfile @@ -0,0 +1,12 @@ +FROM continuumio/miniconda + +COPY pipeline_controller/requirements.txt . +COPY pipeline_controller/installer_Rpackage.R . + +RUN conda create -n pipeline_controller_base python=3.8.2 R=3.6 +SHELL ["conda", "run", "-n", "pipeline_controller_base", "/bin/bash", "-c"] + +RUN pip install -r requirements.txt +RUN Rscript installer_Rpackage.R + +RUN python -c "import flask" \ No newline at end of file diff --git a/pipelines/deg_pipeline/config_807c4dca2e084124a49f45b38d89c100.yaml b/pipelines/deg_pipeline/config_807c4dca2e084124a49f45b38d89c100.yaml deleted file mode 100644 index 600b01b..0000000 --- a/pipelines/deg_pipeline/config_807c4dca2e084124a49f45b38d89c100.yaml +++ /dev/null @@ -1,8 +0,0 @@ -InputFolder: /MainData/rsem_counts -MetaFile: /MainData/annotation_metadata/EPIC_HCvB_metadata_baseline_updated-share.csv -SampleColumnName: HCVB_ID -CondColumnName: DiseaseCourse -Condition1: RR -Condition2: CIS -OutputFolder: ./Output/ -logID: user1 diff --git a/pipelines/deg_pipeline/config_f52225dc61924bbe8efd55ae0eab76aa.yaml b/pipelines/deg_pipeline/config_f52225dc61924bbe8efd55ae0eab76aa.yaml deleted file mode 100644 index 600b01b..0000000 --- a/pipelines/deg_pipeline/config_f52225dc61924bbe8efd55ae0eab76aa.yaml +++ /dev/null @@ -1,8 +0,0 @@ -InputFolder: /MainData/rsem_counts -MetaFile: /MainData/annotation_metadata/EPIC_HCvB_metadata_baseline_updated-share.csv -SampleColumnName: HCVB_ID -CondColumnName: DiseaseCourse -Condition1: RR -Condition2: CIS -OutputFolder: ./Output/ -logID: user1