Merge branch 'develop' into IA-5026-update-helm-and-k8s

Author: LizBaldo

.github/workflows/automerge.yml

@@ -45,12 +45,10 @@ jobs:
       - name: Match the PR to the corresponding team's slack notification channel
         id: prepare-outputs
         run: |
-          if ${{ contains(github.event.pull_request.title, 'WM-') }}; then
-            echo 'notify-failure-channel=dsp-workflows'                 >> $GITHUB_OUTPUT
-          elif ${{ contains(github.event.pull_request.title, 'WX-') }}; then
-            echo 'notify-failure-channel=dsp-workflows'                >> $GITHUB_OUTPUT
-          else ${{ contains(github.event.pull_request.title, 'AJ-') }};
-           echo 'notify-failure-channel=dsp-analysis-journeys'        >> $GITHUB_OUTPUT
+          if ${{ contains(github.event.pull_request.title, 'AN-') }}; then
+            echo 'notify-failure-channel=dsp-analysis-non-prod-alerts' >> $GITHUB_OUTPUT
+          else ${{ contains(github.event.pull_request.title, 'CORE-') }};
+           echo 'notify-failure-channel=dsp-core-services'        >> $GITHUB_OUTPUT
           fi
 
   report-workflow:

.github/workflows/azure_automation_test.yml

@@ -162,7 +162,7 @@ jobs:
     outputs:
       project-name: ${{ steps.gen.outputs.project_name }}
     steps:
-      - uses: 'actions/checkout@v3'
+      - uses: 'actions/checkout@v4'
 
       - name: Generate a random billing project name
         id: 'gen'

.github/workflows/consumer_contract_tests.yml

@@ -94,7 +94,7 @@ jobs:
       fork: ${{ steps.extract-branch.outputs.fork }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Obtain branch properties
         id: extract-branch
@@ -137,7 +137,7 @@ jobs:
       pact-b64: ${{ steps.encode-pact.outputs.pact-b64 }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Run consumer tests
         run: |
           docker run --rm -v $PWD:/working \

.github/workflows/custom_image_generation.yml

@@ -34,7 +34,7 @@ jobs:
       OUTPUT_FILE_RELATIVE_PATH: jenkins/gce-custom-images/output.txt
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - id: 'auth'
       name: 'Authenticate to Google Cloud'
@@ -81,7 +81,7 @@ jobs:
       DATAPROC_IMAGE_BUCKET: gs://leo-dataproc-image-creation-logs
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'

.github/workflows/leo-build-tag-publish-and-run-tests.yml

@@ -32,7 +32,7 @@ env:
   LEO_SWAT_TESTS_RUN_NAME: 'leonardo-swat-tests-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
   BEE_CREATE_RUN_NAME: 'bee-create-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
   BEE_DESTROY_RUN_NAME: 'bee-destroy-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
-  AZURE_SWAT_TESTS_RUN_NAME: 'azure-swat-tests-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+  # AZURE_SWAT_TESTS_RUN_NAME: 'azure-swat-tests-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
 
 concurrency:
   # Don't run this workflow concurrently on the same branch
@@ -93,7 +93,7 @@ jobs:
       - name: Extract branch
         id: extract-branch
         run: |
-          if [[ '${{ github.event_name }}' == 'push' ]]; then
+          if [[ '${{ github.event_name }}' == 'push' || '${{ github.event_name }}' == 'workflow_run' ]]; then
             BRANCH_NAME=${{ github.ref_name }}
           elif [[ '${{ github.event_name }}' == 'pull_request' ]]; then
             BRANCH_NAME=${{ github.head_ref }}
@@ -216,32 +216,33 @@ jobs:
               "custom-version-json": "${{ steps.render-leo-version.outputs.custom-version-json }}"
             }'
 
-  azure-automation-tests:
-    runs-on: ubuntu-latest
-    needs: [ leo-build-tag-publish-job, report-to-sherlock, init-github-context, create-bee-workflow ]
-    permissions:
-      contents: 'read'
-      id-token: 'write'
-    steps:
-      - name: dispatch to azure automation test workflow
-        uses: broadinstitute/[email protected]
-        with:
-          run-name: "${{ env.AZURE_SWAT_TESTS_RUN_NAME }}"
-          workflow: .github/workflows/azure_automation_test.yml
-          repo: DataBiosphere/leonardo
-          ref: refs/heads/develop
-          token: ${{ env.TOKEN }}
-          # this is not actually 85mins, the timeout is not working correctly and is summing from other jobs
-          wait-for-completion-timeout: 85m
-          inputs: '{
-            "terra-env": "${{ needs.init-github-context.outputs.test-env }}",
-            "automation-branch": "${{ needs.init-github-context.outputs.automation-branch }}",
-            "app-version": "${{ needs.leo-build-tag-publish-job.outputs.tag }}",
-            "bee-name": "${{ env.BEE_NAME }}",
-            "is-triggered-from-pr-dispatch": "true",
-            "run-name": "${{ env.AZURE_SWAT_TESTS_RUN_NAME }}"
-            }'
-          wait-for-completion: true
+## !! Commenting out until when/if the azure tests are necessary again...
+#  azure-automation-tests:
+#    runs-on: ubuntu-latest
+#    needs: [ leo-build-tag-publish-job, report-to-sherlock, init-github-context, create-bee-workflow ]
+#    permissions:
+#      contents: 'read'
+#      id-token: 'write'
+#    steps:
+#      - name: dispatch to azure automation test workflow
+#        uses: broadinstitute/[email protected]
+#        with:
+#          run-name: "${{ env.AZURE_SWAT_TESTS_RUN_NAME }}"
+#          workflow: .github/workflows/azure_automation_test.yml
+#          repo: DataBiosphere/leonardo
+#          ref: refs/heads/develop
+#          token: ${{ env.TOKEN }}
+#          # this is not actually 85mins, the timeout is not working correctly and is summing from other jobs
+#          wait-for-completion-timeout: 85m
+#          inputs: '{
+#            "terra-env": "${{ needs.init-github-context.outputs.test-env }}",
+#            "automation-branch": "${{ needs.init-github-context.outputs.automation-branch }}",
+#            "app-version": "${{ needs.leo-build-tag-publish-job.outputs.tag }}",
+#            "bee-name": "${{ env.BEE_NAME }}",
+#            "is-triggered-from-pr-dispatch": "true",
+#            "run-name": "${{ env.AZURE_SWAT_TESTS_RUN_NAME }}"
+#            }'
+#          wait-for-completion: true
 
   leo-swat-test-job:
     strategy:
@@ -298,7 +299,7 @@ jobs:
     needs:
       - init-github-context
       - leo-swat-test-job
-      - azure-automation-tests
+     # - azure-automation-tests
     if: ${{ needs.init-github-context.outputs.delete-bee && always() }} # always run to confirm bee is destroyed unless explicitly requested not to
     permissions:
       contents: 'read'

.github/workflows/publish_java_client.yml

@@ -29,7 +29,8 @@ jobs:
       SBT_OPTS: -Xmx3g
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - uses: sbt/setup-sbt@v1
 
       # coursier cache action caches both coursier and sbt caches
       - name: coursier-cache-action

.github/workflows/tag.yml

@@ -46,7 +46,7 @@ jobs:
       new-tag: ${{ steps.tag.outputs.new_tag }}
     steps:
     - name: Checkout current code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ inputs.ref }}
         token: ${{ secrets.BROADBOT_TOKEN }} # this allows the push to succeed later

.github/workflows/trivy.yml

@@ -7,7 +7,7 @@ jobs:
     name: DSP AppSec Trivy check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       # The Dockerfile copies this, so it needs
       # to exist for the build to succeed

.github/workflows/unit_test.yml

@@ -21,7 +21,7 @@ jobs:
 
     services:
       mysql:
-        image: mysql/mysql-server:5.6
+        image: mysql:8.4
         env:
           MYSQL_ROOT_PASSWORD: leonardo-test
           MYSQL_USER: leonardo-test
@@ -36,7 +36,8 @@ jobs:
           - 3307:3306
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+    - uses: sbt/setup-sbt@v1
 
     - name: Set up JDK 17
       uses: actions/setup-java@v3

.github/workflows/verify_consumer_contracts.yml

@@ -107,7 +107,7 @@ jobs:
 
     steps:
     - name: Checkout current code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

CODEOWNERS

@@ -0,0 +1,6 @@
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence,
+# @broadinstitute/dsp-analysis will be requested for
+# review when someone opens a pull request.
+* @DataBiosphere/dsp-analysis
+

Dockerfile

@@ -27,7 +27,7 @@ ENV HELM_DEBUG 1
 ENV TERRA_APP_SETUP_VERSION 0.1.0
 ENV TERRA_APP_VERSION 0.5.0
 # This is galaxykubeman, which references Galaxy
-ENV GALAXY_VERSION 2.9.0
+ENV GALAXY_VERSION 2.10.0
 ENV NGINX_VERSION 4.3.0
 # If you update this here, make sure to also update reference.conf:
 ENV CROMWELL_CHART_VERSION 0.2.523

automation/src/test/resources/reference.conf

@@ -1,13 +1,13 @@
 leonardo {
-    rImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.5"
+    rImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.2.6"
     pythonImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.5"
-    hailImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.10"
-    gatkImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.7"
-    aouImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.12"
+    hailImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-hail:1.1.12"
+    gatkImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.8"
+    aouImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.14"
     baseImageUrl = "us.gcr.io/broad-dsp-gcr-public/terra-jupyter-base:1.1.3"
     gcrWelderUri = "us.gcr.io/broad-dsp-gcr-public/welder-server"
     dockerHubWelderUri = "broadinstitute/welder-server"
-    rstudioBioconductorImageUrl = "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.19.0"
+    rstudioBioconductorImageUrl = "us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.20.1"
     //each fiab will have a unique topic name sourced from leonardo.conf, this is never published to in automation tests to ensure pub/sub components can auth properly, but never read from to avoid conflicts.
     topicName = "leonardo-pubsub-test"
     location = "us-central1-a"

automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/RuntimeFixtureSpec.scala

@@ -227,7 +227,7 @@ object RuntimeFixtureSpec {
       case CloudService.Dataproc =>
         RuntimeConfigRequest.DataprocConfig(
           numberOfWorkers = Some(0),
-          masterDiskSize = Some(DiskSize(130)),
+          masterDiskSize = Some(DiskSize(150)),
           masterMachineType = Some(MachineTypeName("n1-standard-8")),
           workerMachineType = Some(MachineTypeName("n1-standard-8")),
           workerDiskSize = None,

automation/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/runtimes/RuntimeDataprocSpec.scala

@@ -95,7 +95,7 @@ class RuntimeDataprocSpec extends BillingProjectFixtureSpec with ParallelTestExe
             RuntimeConfigRequest.DataprocConfig(
               Some(2),
               Some(MachineTypeName("n1-standard-4")),
-              Some(DiskSize(130)),
+              Some(DiskSize(150)),
               Some(MachineTypeName("n1-standard-4")),
               Some(DiskSize(150)),
               None,
@@ -158,7 +158,7 @@ class RuntimeDataprocSpec extends BillingProjectFixtureSpec with ParallelTestExe
         RuntimeConfigRequest.DataprocConfig(
           Some(2),
           Some(MachineTypeName("n1-standard-4")),
-          Some(DiskSize(130)),
+          Some(DiskSize(150)),
           Some(MachineTypeName("n1-standard-4")),
           Some(DiskSize(150)),
           None,

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/diskRoutesModels.scala

@@ -42,7 +42,8 @@ final case class ListPersistentDiskResponse(id: DiskId,
                                             size: DiskSize,
                                             diskType: DiskType,
                                             blockSize: BlockSize,
-                                            labels: LabelMap
+                                            labels: LabelMap,
+                                            workspaceId: Option[WorkspaceId]
 )
 
 final case class GetPersistentDiskResponse(id: DiskId,
@@ -57,7 +58,8 @@ final case class GetPersistentDiskResponse(id: DiskId,
                                            diskType: DiskType,
                                            blockSize: BlockSize,
                                            labels: LabelMap,
-                                           formattedBy: Option[FormattedBy]
+                                           formattedBy: Option[FormattedBy],
+                                           workspaceId: Option[WorkspaceId]
 )
 
 final case class GetPersistentDiskV2Response(id: DiskId,

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/runtimeModels.scala

@@ -525,17 +525,26 @@ object WelderAction extends Enum[WelderAction] {
   case object DisableDelocalization extends WelderAction
 }
 
-final case class MemorySize(bytes: Long) extends AnyVal {
+final case class MemorySizeBytes(bytes: Long) extends AnyVal {
   override def toString: String = bytes.toString + "b"
 }
-object MemorySize {
+object MemorySizeBytes {
   val kbInBytes = 1024
   val mbInBytes = 1048576
   val gbInBytes = 1073741824
 
-  def fromKb(kb: Double): MemorySize = MemorySize((kb * kbInBytes).toLong)
-  def fromMb(mb: Double): MemorySize = MemorySize((mb * mbInBytes).toLong)
-  def fromGb(gb: Double): MemorySize = MemorySize((gb * gbInBytes).toLong)
+  def fromKb(kb: Double): MemorySizeBytes = MemorySizeBytes((kb * kbInBytes).toLong)
+  def fromMb(mb: Double): MemorySizeBytes = MemorySizeBytes((mb * mbInBytes).toLong)
+  def fromGb(gb: Double): MemorySizeBytes = MemorySizeBytes((gb * gbInBytes).toLong)
+}
+
+final case class MemorySizeMegaBytes(megabytes: Long) extends AnyVal {
+  override def toString: String = megabytes.toString + "m"
+}
+object MemorySizeMegaBytes {
+  val mbInBytes = 1048576
+
+  def fromB(b: Double): MemorySizeMegaBytes = MemorySizeMegaBytes((b / mbInBytes).toLong)
 }
 
 /**
@@ -548,9 +557,10 @@ object MemorySize {
  * Note that the memory limit includes all the sub-procesess of the Notebook server including the
  * Notebook kernel and the Spark driver process, if any.
  */
-final case class RuntimeResourceConstraints(memoryLimit: MemorySize,
-                                            totalMachineMemory: MemorySize,
-                                            driverMemory: Option[MemorySize]
+final case class RuntimeResourceConstraints(memoryLimit: MemorySizeBytes,
+                                            shmSize: MemorySizeMegaBytes,
+                                            totalMachineMemory: MemorySizeBytes,
+                                            driverMemory: Option[MemorySizeBytes]
 )
 
 final case class RuntimeMetrics(cloudContext: CloudContext,