Merge branch 'main' of https://github.com/NIGMS/NIGMS-Sandbox into Update_AWSBatch

Author: aolveraNIH

.github/workflows/check-links.yaml

@@ -30,5 +30,5 @@ jobs:
         run: |
           python3 ${GITHUB_WORKSPACE}/reusable-workflow-repo/.github/workflows/check_links.py
         env:
-          LINK_IGNORE_LIST: https://www.sciencedirect.com,https://portlandpress.com,cloud.google.com,aws.amazon.com
-          PAT: ${{ secrets.PAT }}
+          LINK_IGNORE_LIST: "https://www.sciencedirect.com,https://portlandpress.com,cloud.google.com,aws.amazon.com,https://journals.biologists.com/bio/article-pdf/5/8/1134/1114440/bio020065.pdf,https://onlinelibrary.wiley.com/doi/10.1111/1755-0998.13593,https://www.ensembl.org/,https://github.com/NIGMS/NIGMS-Sandbox/blob/main/README.md,https://github.com/rnaseqprok,https://github.com/rnaseqprok/rnaseqprok/blob/master/CITATIONS.md,https://github.com/NIGMS/NIGMS-Sandbox/blob/main/README.md#available-modules,https://github.com/nf-core/rnaseq/blob/4c27ef5610c87db00c3c5a3eed10b1d161abf575/conf/base.config#L18,https://snakemake.readthedocs.io/en/stable/executor_tutorial/google_lifesciences.html,https://nersc.gitlab.io/development/shifter/how-to-use/,http://bioconductor.org/packages/release/BiocViews.html#___OrgDb,https://www.bioconductor.org/packages/release/bioc/vignettes/methylKit/inst/doc/methylKit.html#4_Annotating_differentially_methylated_bases_or_regions,https://docs.qiime2.org/2024.2/install/native/#qiime-2-metagenome-distribution,https://www.researchgate.net/figure/Diagrams-of-the-BRCA1-and-BRCA2-genes-indicating-the-position-of-pathogenic-variants_fig2_321205153,https://www.fda.gov/drugs/biomarker-qualification-program/about-biomarkers-and-qualification,https://github.com/NIGMS/NIGMS-Sandbox/blob/main/README.md#available-modules,https://www.cambridge.org/highereducation/books/introduction-to-applied-linear-algebra/4D69AF22E38303FE20FFEEFDCE0E7F96#overview,https://doi.org/10.1089/wound.2019.1030,https://bio.tools/spades,https://en.wikipedia.org/wiki/Bash_(Unix_shell),https://github.com/NIGMS/NIGMS-Sandbox/blob/main/docs/HowToCreateVertexAINotebooks.md#:~:text=Select%20Advanced%20Options%20at%20the%20bottom%20of%20the%20New%20Instance,https://labormarketinfo.edd.ca.gov/data/unemployment-and-labor-force.html,https://github.com/FelixKrueger/Bismark/blob/master/docs/bismark/alignment.md#bismark-bamsam-output-default"
+          PAT: ${{ secrets.PAT }}

.github/workflows/check_links.py

@@ -141,13 +141,14 @@ def check_link(link):
     elif print_valid:
         print(
             loc + " , " + link + " , Valid")
-    return True
+    return False
 
 
 if __name__ == "__main__":
     err = 0
     print("Directory is "+path)
     # Loop through all files in path
+
     for root, dirs, files in os.walk(path):
         for file in files:
             #  only read file that match template ( txt, md or python notebook)

.github/workflows/lint.py

@@ -9,6 +9,10 @@ def clean_notebook(file_path):
 
     # Clean cells
     for cell in notebook.cells:
+        # skip cells with a quiz or no clean tag
+        if 'metadata' in cell and 'tags' in cell['metadata'] and (
+                'quiz' in cell['metadata']['tags'] or 'noclean' in cell['metadata']['tags']):
+            continue
         if 'outputs' in cell:
             cell['outputs'] = []
         if 'execution_count' in cell:
@@ -18,7 +22,12 @@ def clean_notebook(file_path):
 
     # Clean notebook metadata
     if 'metadata' in notebook:
-        notebook['metadata'] = {}
+        new_metadata = {}
+        if 'language_info' in notebook['metadata']:
+            new_metadata['language_info'] = notebook['metadata']['language_info']
+        if 'kernelspec' in notebook['metadata']:
+            new_metadata['kernelspec'] = notebook['metadata']['kernelspec']
+        notebook['metadata'] = new_metadata
 
     with open(file_path, 'w', encoding='utf-8') as f:
         nbformat.write(notebook, f)
@@ -39,7 +48,7 @@ def delete_checkpoints_dirs(root_dir):
 
 if __name__ == "__main__":
     # Change this to the directory containing your notebooks
-    notebook_dir = '../../'
+    notebook_dir = '.'
 
     for root, dirs, files in os.walk(notebook_dir):
         for file in files:

.github/workflows/notebook-lint.yaml

@@ -38,13 +38,16 @@ jobs:
           pip3 install nbformat
 
       - name: Notebook Linting
-        working-directory: reusable-workflow-repo/.github/workflows/
         run: |
-          python3 lint.py
+          python3 ${GITHUB_WORKSPACE}/reusable-workflow-repo/.github/workflows/lint.py
+
+      - name: Remove workflow repo
+        run: |
+          rm ${GITHUB_WORKSPACE}/reusable-workflow-repo -r
 
       - name: Commit changes
         uses: EndBug/add-and-commit@v9
         with:
           author_name: github-action
           author_email: cbiit-github-action@github.com
-          message: 'Github Action: Refresh stats'
+          message: 'Github Action: Lint Notebooks'

CheckLinks.py

@@ -6,40 +6,45 @@ AWS offers different types of Notebook instances: SageMaker Studio, SageMaker No
 
 1. In AWS console, type **Amazon SageMaker** in search bar and click it:
 
-    ![selectsagemaker](/images/images_for_creating_AWS_notebooks/Screenshot1.png)
+  ![selectsagemaker](/images/images_for_creating_AWS_notebooks/Screenshot1.png)
 
 2. On the left side bar click **Notebooks**:
 
-    ![Notebooks](/images/images_for_creating_AWS_notebooks/Screenshot2.png)
+  ![Notebooks](/images/images_for_creating_AWS_notebooks/Screenshot2.png)
 
 3. Click **Create notebook instance**:
 
-    ![create_new_notebook](/images/images_for_creating_AWS_notebooks/Screenshot3.png)
+  ![create_new_notebook](/images/images_for_creating_AWS_notebooks/Screenshot3.png)
 
 4. Give a name to your notebook. Choose a notebook instance type based on needs, Amazon Linux 2 as platform identifier, volume. Optional, create idle-shut by selecting create new lifecycle configuration and copy and paste [idle-shutdown.sh](/idle-shutdown.sh) and create configuration. Then click Create notebook instance:
 
-    ![environment](/images/images_for_creating_AWS_notebooks/Screenshot4.png)
+  ![environment](/images/images_for_creating_AWS_notebooks/Screenshot4.png)
 
 5. It will take about 5 minutes, your notebook instance to get ready:
 
-    ![pending](/images/images_for_creating_AWS_notebooks/Screenshot5.png)
+  ![pending](/images/images_for_creating_AWS_notebooks/Screenshot5.png)
 
 
 6. **Open Jupyter Lab**:
 
-    ![jupyterlab](/images/images_for_creating_AWS_notebooks/Screenshot6.png)
+  ![jupyterlab](/images/images_for_creating_AWS_notebooks/Screenshot6.png)
+
+7. Clone an appropriate repository from GitHub.com:
+
+  ![upload](/images/images_for_creating_AWS_notebooks/Screenshot7.png)
 
-7. Upload your jupyter notebook
+8. Select a notebook and then kernel:
 
-    ![upload](/images/images_for_creating_AWS_notebooks/Screenshot7.png)
+  ![kernel](/images/images_for_creating_AWS_notebooks/Screenshot8.png)
 
-8. Select the kernel:
+9.  To clone in your git repositories you can either use the git UI built in or by using the terminal. Using the git UI select **Clone a Repository** then paste the repo url and click **Clone**. To clone via the terminal make sure that you are in the SageMaker directory then use the command git clone [repo url]. Once you have done this you should see the repo 
 
-    ![kernel](/images/images_for_creating_AWS_notebooks/Screenshot8.png)
+  ![Clone_UI](/images/images_for_creating_AWS_notebooks/Clone_UI.png)
+  ![Clone_Terminal](/images/images_for_creating_AWS_notebooks/Clone_Terminal.png)
 
-9. **IMPORTANT**: **Stop** the kernel at end of your work to avoid getting charged.
+10. **IMPORTANT**: **Stop** the kernel at end of your work to avoid getting charged.
 
-    ![resize image](/images/images_for_creating_AWS_notebooks/Screenshot9.png)
+  ![resize image](/images/images_for_creating_AWS_notebooks/Screenshot9.png)
 
 
 
@@ -59,15 +64,15 @@ AWS offers different types of Notebook instances: SageMaker Studio, SageMaker No
 
 4. Select **Set up for single user (Quick setup)** and then click **Set up**:
 
-    ![Domain Setup](/images/images_for_creating_AWS_notebooks/Screenshot12.png)
+  ![Domain Setup](/images/images_for_creating_AWS_notebooks/Screenshot12.png)
 
 5. It will take a few minutes for the domain to setup:
 
-    ![pending](/images/images_for_creating_AWS_notebooks/Screenshot13.png)
+  ![pending](/images/images_for_creating_AWS_notebooks/Screenshot13.png)
 
 6. Once the SageMaker Domain is setup, click **Studio** on the left sidebar:
 
-    ![Studio](/images/images_for_creating_AWS_notebooks/Screenshot14.png)
+  ![Studio](/images/images_for_creating_AWS_notebooks/Screenshot14.png)
 
 7. Make sure that **Select Domain** and **Select user profile** contain domain and user then click **Open Studio**:
 
@@ -134,11 +139,15 @@ AWS offers different types of Notebook instances: SageMaker Studio, SageMaker No
   ![JupyterLab Space](/images/images_for_creating_AWS_notebooks/Screenshot29.png)
   ![JupyterLab Space](/images/images_for_creating_AWS_notebooks/Screenshot30.png)
 
-6. After working go back to Code Editor and stop the now unused instance by clicking ***Stop***:
+6. Once in code editor clone in your repository. This can be done by selecting the git icon, select **Clone Repository**, copy the web url into the box and then select **Clone from URL**. Alternatively you can open a terminal then use the command git clone web url. Once you do this cd into the cloned repository. If it doesn't show up in your folders select the file icon open folder and select the repository you cloned in. 
+
+  ![Clone For Code Editor](/images/images_for_creating_AWS_notebooks/Clone_CodeEditor.png)
+
+7. After working go back to Code Editor and stop the now unused instance by clicking ***Stop***:
 
   ![JupyterLab Space](/images/images_for_creating_AWS_notebooks/Screenshot31.png)
 
-7. Wait for the Status to show as Stopped:
+8. Wait for the Status to show as Stopped:
 
   ![JupyterLab Space](/images/images_for_creating_AWS_notebooks/Screenshot32.png)
 

docs/HowToCreateAWSSagemakerNotebooks.md

@@ -0,0 +1,41 @@
+# Set Up a JupyterLab with a Custom Docker Image
+
+1. In Amazon SageMaker AI, **Create domain**:
+
+![](/images/images_custom_kernel/Capture1.PNG)
+
+2. Leave the **Quick setup** and click **Set up**:
+
+![](/images/images_custom_kernel/Capture2.PNG)
+
+3. After having your domain ready, go to **Environment** tab and click **Attach image**:
+
+![](/images/images_custom_kernel/Capture3.PNG)
+
+4. Under **New image**, enter and ECR image URL, click **Next**:
+
+![](/images/images_custom_kernel/Capture4.PNG)
+
+5. Give a name to your image, select **JupyterLab image**, and **Submit**:
+
+![](/images/images_custom_kernel/Capture5.PNG)
+
+6. In **Studio** tab, **Select Domain** and then **Open Studio**:
+
+![](/images/images_custom_kernel/Capture6.PNG)
+
+7. Select **Jupyterlab**, and then click **Create JupyterLab Space**:
+
+![](/images/images_custom_kernel/Capture7.PNG)
+
+8. Give a name, and then **Create space**:
+
+![](/images/images_custom_kernel/Capture8.PNG)
+
+9. Select the custom image, instance type, and then **Run space**:
+
+![](/images/images_custom_kernel/Capture9.PNG)
+
+10. **Open JupyterLab**:
+
+![](/images/images_custom_kernel/Capture10.PNG)

docs/HowToCreateJupyterlabWithCustomImage.md

@@ -2,7 +2,7 @@
 
 Jupyter lab allows you to create a new console window within its layout. This is also possible on Jupyter lab on AWS SageMaker. Adding a terminal console window where there is a jupyter notebook tab is helpful when needing to run command line code. 
 
-## 1) In Jupyter lab within a jupyter notebook, right click on any whitespace in the jupyter notebook. 
+## 1) In Jupyter lab within a Jupyter notebook, right click on any whitespace in the notebook. 
 
 ![Right_click](/images/images_for_creating_new_console/Right_click_notebook.png)
 

docs/HowToCreateNewTerminalConsoleJupyterLab.md

@@ -29,7 +29,37 @@
 
 9. The remaining sections can be left as default for our purposes.  Further details can be found in the official documentation: [Vertex AI Documentation](https://cloud.google.com/vertex-ai/docs/workbench/instances/create)
 
-### 2. Spin up a User-Managed Notebook Instance
+### 2. Spin up Instance from a Container
+1. Start by clicking the `hamburger menu` (the three horizontal lines in the top left of your console). Go to `Artificial Intelligence > Vertex AI > Workbench`. 
+
+![screenshot showing how to select Vertex AI workbench](/images/1_select_vertexAI.png)
+
+2. If not already selected, click **Instances**, then **Instances**
+3. Click **+ Create New**
+
+![image showing how to select instance](/images/images_for_creating_GCP_instances/2_select_workbench_instance.png)
+
+4. Select **Advanced Options** at the bottom of the **New Instance** pop-up window
+5. Provide a name for your new instance using letters, numbers, and hyphens (-). Select a region and zone for the new instance. For best network performance, select the region that is geographically closest to you.  Click **Continue**
+
+![image showing to select advanced options](/images/images_for_creating_GCP_instances/3_select_advanced_options.png)
+
+6. On the Environment screen, select "Use custom container" and then enter your container name. \
+   Click **Continue**.
+
+![image showing to select environment](/images/images_for_creating_GCP_instances/custom_container.png) 
+
+7. On the Machine type screen, select the desired number of CPUs/GPUs. This is usually specified by the tutorial you are completing.
+
+![image showing machine type selection](/images/images_for_creating_GCP_instances/5_instance_machine_type.png)
+
+8. On the same screen, verify that **Enable Idle Shutdown** is selected and specify the idle minutes for shutdown. This means it will shutdown automatically after this many minutes. We recommend 30 minutes. Click **Create**.  It will take a few minutes for your instance to spin up.
+
+![image showing idle shutdown selection](/images/images_for_creating_GCP_instances/6_instance_idle_shutdown.png) 
+
+9. The remaining sections can be left as default for our purposes.  Further details can be found in the official documentation: [Vertex AI Documentation](https://cloud.google.com/vertex-ai/docs/workbench/instances/create)
+
+### 3. Spin up a User-Managed Notebook Instance
 1. Start by clicking the `hamburger menu` (the three horizontal lines in the top left of your console). Go to `Artificial Intelligence > Vertex AI > Workbench`. 
 
 ![screenshot showing how to select Vertex AI workbench](/images/1_select_vertexAI.png)