feat(quickstart): migrate to compose profile and other improv.

.github/workflows/quickstart-test.yml

@@ -0,0 +1,163 @@
+name: Quickstart Test
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "2 0 * * *"
+  push:
+    branches:
+      - master
+    paths:
+      - "metadata-ingestion/src/datahub/cli/docker_cli.py"
+      - "metadata-ingestion/src/datahub/cli/docker_check.py"
+      - "metadata-ingestion/src/datahub/cli/quickstart_versioning.py"
+      - ".github/workflows/quickstart-test.yml"
+      - "docker/quickstart/docker-compose.quickstart-profile.yml"
+  pull_request:
+    paths:
+      - "metadata-ingestion/src/datahub/cli/docker_cli.py"
+      - "metadata-ingestion/src/datahub/cli/docker_check.py"
+      - "metadata-ingestion/src/datahub/cli/quickstart_versioning.py"
+      - ".github/workflows/quickstart-test.yml"
+      - "docker/quickstart/docker-compose.quickstart-profile.yml"
+
+jobs:
+  test-quickstart-pr:
+    name: Quickstart Smoke Tests
+    if: ${{ github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          ./gradlew :metadata-ingestion:install
+
+      - name: Run quickstart
+        run: |
+          source metadata-ingestion/venv/bin/activate
+          # Use compose and version mapping files from PR
+          export FORCE_LOCAL_QUICKSTART_MAPPING=docker/quickstart/quickstart_version_mapping.yaml
+          datahub docker quickstart -f docker/quickstart/docker-compose.quickstart-profile.yml
+          docker images
+
+      - name: Run quickstart check
+        run: |
+          source metadata-ingestion/venv/bin/activate
+          datahub docker check
+
+      - name: Smoke test
+        if: false
+        env:
+          RUN_QUICKSTART: false
+          DATAHUB_VERSION: head
+          CLEANUP_DATA: "false"
+          TEST_STRATEGY: cypress
+          BATCH_COUNT: 30
+          BATCH_NUMBER: 2
+        run: |
+          echo "$DATAHUB_VERSION"
+          ./gradlew --stop
+          ./smoke-test/smoke.sh
+
+      - name: Run quickstart check
+        run: |
+          source metadata-ingestion/venv/bin/activate
+          datahub docker check
+
+  test-quickstart-pr-compatiblity-checks:
+    name: Quickstart compatiliby Tests
+    if: ${{ github.event_name == 'pull_request' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: install older datahub cli
+        run: |
+          pip install acryl-datahub==1.0
+
+      - name: bring up datahub using old cli
+        run: |
+          datahub docker quickstart
+          datahub docker check
+          datahub docker ingest-sample-data
+          datahub docker check
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          ./gradlew :metadata-ingestion:install
+
+      - name: Run quickstart
+        run: |
+          source metadata-ingestion/venv/bin/activate
+
+          # Use compose and version mapping files from PR
+          export FORCE_LOCAL_QUICKSTART_MAPPING=docker/quickstart/quickstart_version_mapping.yaml
+          if datahub docker quickstart -f docker/quickstart/docker-compose.quickstart-profile.yml; then
+            echo "Command succeeded when it should have failed"
+            exit 1
+          else
+            echo "Command failed as expected"
+          fi
+
+      - name: Backup, nuke and restart and restore
+        run: |
+          source metadata-ingestion/venv/bin/activate
+          echo "Running backup"
+          datahub docker quickstart --backup
+          echo "Running nuke"
+          datahub docker nuke
+          # Use compose and version mapping files from PR
+          export FORCE_LOCAL_QUICKSTART_MAPPING=docker/quickstart/quickstart_version_mapping.yaml
+          echo "Running quickstart"
+          datahub docker quickstart -f docker/quickstart/docker-compose.quickstart-profile.yml
+          echo "Running restore"
+          echo "y" | datahub docker quickstart --restore
+          datahub docker check
+
+  test-quickstart-regression:
+    name: Quickstart regression tests
+    if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # TODO: once more supported versions are aded, this should become list last n versions
+        datahub_version: ["v1.0.0", "v1.1.0", "head"]
+      fail-fast: false
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: install datahub cli
+        run: |
+          pip install acryl-datahub
+
+      - name: bring up datahub using old cli
+        run: |
+          datahub docker quickstart --version ${{ matrix.datahub_version }}
+          datahub docker check
+          datahub docker ingest-sample-data
+          datahub docker check
+
+      - name: Backup, nuke and restart and restore
+        run: |
+          echo "Running backup"
+          datahub docker quickstart --backup
+          echo "Running nuke"
+          datahub docker nuke
+          datahub docker quickstart
+          echo "Running restore"
+          echo "y" | datahub docker quickstart --restore
+          datahub docker check

.github/workflows/verify-quickstart-compose.yml

@@ -0,0 +1,30 @@
+name: Verify Quickstart Compose
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+  pull_request:
+jobs:
+  verify-quickstart-compose-updated:
+    name: Verify quickstart compose file is up-to-date
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Generate quickstart compose file
+        run: |
+          ./gradlew :docker:generateQuickstartComposeConfig
+
+      - name: Verify generated file
+        # If there are build changes, then the generated file will be different from the one in the PR
+        run: |
+          git diff --exit-code docker/quickstart/docker-compose.quickstart-profile.yml
+
+      - name: Validation
+        if: failure()
+        run: |
+          echo "Build file changes are detected. Run\n ./gradlew :docker:generateQuickstartComposeConfig \n and commit the generated docker/quickstart/docker-compose.quickstart-profile.yml file"
+          exit 1

docker/build.gradle

@@ -4,6 +4,7 @@ plugins {
 }
 import com.avast.gradle.dockercompose.tasks.ComposeUp
 import com.avast.gradle.dockercompose.tasks.ComposeDownForced
+import org.yaml.snakeyaml.Yaml
 
 apply from: "../gradle/versioning/versioning.gradle"
 
@@ -39,6 +40,13 @@ ext {
                             ':datahub-actions',
                     ]
             ],
+            'quickstartCLI': [
+                    profile: 'quickstart',
+                    modules: python_services_modules + backend_profile_modules + [
+                            ':datahub-frontend',
+                            ':datahub-actions',
+                    ]
+            ],
             'quickstartDebug': [
                     profile: 'debug',
                     modules: python_services_modules + backend_profile_modules + [':datahub-frontend', ':datahub-actions'],
@@ -162,6 +170,7 @@ dockerCompose {
             environment.put "METADATA_TESTS_ENABLED", "true"
             environment.put "DATAHUB_REPO", "${docker_registry}"
 
+
             // Additional environment variables if specified
             if (config.additionalEnv) {
                 config.additionalEnv.each { key, value ->
@@ -194,6 +203,50 @@ dockerCompose {
 }
 
 
+task generateQuickstartComposeConfig {
+    description = "Generates the docker-compose config output for quickstart configuration for use with oss quickstart cli"
+    group = "docker"
+
+    doLast {
+        def composeUpTask = tasks.named("quickstartCLIComposeUp")
+
+        // Some of these details task gleaned from the docker-compose plugin source code
+        // https://github.com/avast/gradle-docker-compose-plugin/blob/main/src/main/groovy/com/avast/gradle/dockercompose/ComposeExecutor.groovy
+        def composeExecutor = composeUpTask.get().getComposeExecutor().get()
+        composeExecutor.parameters.environment.put("DATAHUB_VERSION", '__VERSION__')
+        composeExecutor.parameters.environment.put("METADATA_SERVICE_AUTH_ENABLED", 'false')
+
+        def configOutput = composeExecutor.execute('config')
+
+        // The `config` returns a fully resolved compose file. We need to replace the DATAHUB_VERSION and HOME with values not available at build time
+        configOutput = configOutput.replace('__VERSION__', '${DATAHUB_VERSION}')
+        configOutput = configOutput.replace(System.getenv('HOME'), '${HOME}')
+
+        // Parse configOutput as yaml and add UI_INGESTION_DEFAULT_CLI_VERSION as an env var under datahub-gms-quickstart.environment key
+        // This needs to be matched with the version of the CLI used to launch quickstart
+        def yaml = new Yaml()
+        def config = yaml.load(configOutput)
+        config['services']['datahub-gms-quickstart']['environment']['UI_INGESTION_DEFAULT_CLI_VERSION'] = '${UI_INGESTION_DEFAULT_CLI_VERSION}'
+
+        // Configure YAML dump options to match Docker Compose format
+        def dumpOptions = new org.yaml.snakeyaml.DumperOptions()
+        dumpOptions.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK)
+        dumpOptions.setIndent(2)
+        dumpOptions.setWidth(120)
+        yaml = new Yaml(dumpOptions)
+        configOutput = yaml.dump(config)
+
+        configOutput = "# This file is generated as part of build process. If any build changes cause this file to be modified, please check in the generated file\n" + configOutput
+
+        def outputFile = project.file("${project.projectDir}/quickstart/docker-compose.quickstart-profile.yml")
+        outputFile.getParentFile().mkdirs()
+
+        outputFile.text = configOutput
+
+        logger.lifecycle("quickstart Docker Compose config saved to: ${outputFile.absolutePath}")
+    }
+}
+
 // Register all quickstart tasks
 quickstart_configs.each { taskName, config ->
     tasks.register("prepareAll${taskName}"){
@@ -219,6 +272,7 @@ quickstart_configs.each { taskName, config ->
         def taskSuffix = config.isDebug? 'debug' : ''
         dependsOn(config.modules.collect { it + ":generateBakeSnippet${taskSuffix}" })
         dependsOn(tasks.getByName("prepareAll${taskName}"))
+        dependsOn tasks.named("generateQuickstartComposeConfig")
 
         def jsonFile = new File(rootProject.buildDir, "bake-spec-${taskName}.json")
 

docker/profiles/docker-compose.gms.yml

@@ -71,6 +71,8 @@ x-datahub-system-update-service: &datahub-system-update-service
     ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS: ${ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS:-3}
   volumes:
     - ${HOME}/.datahub/plugins:/etc/datahub/plugins
+  labels:
+    datahub_setup_job: true
 
 x-datahub-system-update-service-dev: &datahub-system-update-service-dev
   <<: *datahub-system-update-service
@@ -106,6 +108,7 @@ x-datahub-gms-service: &datahub-gms-service
     STRICT_URN_VALIDATION_ENABLED: ${STRICT_URN_VALIDATION_ENABLED:-true}
     ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}
     THEME_V2_DEFAULT: ${THEME_V2_DEFAULT:-true}
+    METADATA_SERVICE_AUTH_ENABLED: ${METADATA_SERVICE_AUTH_ENABLED:-true}
   healthcheck:
     test: curl -sS --fail http://datahub-gms:${DATAHUB_GMS_PORT:-8080}/health
     start_period: 90s
@@ -225,6 +228,10 @@ services:
         condition: service_completed_successfully
       kafka-setup:
         condition: service_completed_successfully
+      opensearch:
+        condition: service_healthy
+      mysql:
+        condition: service_healthy
   system-update-quickstart-cassandra:
     <<: *datahub-system-update-service
     profiles: