Merge branch '7.6.x' into 7.7.x by wadhwa1

semaphore-agent-production[bot] · semaphore-agent-production[bot] · commit 7e3bdc2b03ef · 2025-04-03T07:20:47.000Z
diff --git a/.semaphore/cp_dockerfile_build.yml b/.semaphore/cp_dockerfile_build.yml
@@ -7,7 +7,7 @@ version: v1.0
 name: build-test-release
 agent:
   machine:
-    type: s1-prod-ubuntu20-04-amd64-1
+    type: s1-prod-ubuntu24-04-amd64-1
 
 fail_fast:
   cancel:
@@ -78,6 +78,28 @@ global_job_config:
       - export LATEST_TAG=$BRANCH_TAG-latest
       - export DOCKER_UPSTREAM_TAG="$LATEST_TAG"
       - export DOCKER_REPOS="confluentinc/kafka-streams-examples"
+      - export COMMUNITY_DOCKER_REPOS=""
+      - |
+        if [[ $SKIP_COMMUNITY == "True" ]]; then
+          # Filter out community repos from DOCKER_REPOS
+          DOCKER_REPOS=$(comm -23 <(echo "$DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs)
+          export DOCKER_REPOS
+          echo "DOCKER_REPOS after skipping community images - $DOCKER_REPOS"
+
+          # Set Maven arguments for skipping community modules
+          export MAVEN_EXTRA_ARGS=""
+
+          # Check if current DOCKER_IMAGE is in community repos, skip job execution
+          for skip_repo in $COMMUNITY_DOCKER_REPOS; do
+            if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
+              echo "Skipping build for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
+              export SEMAPHORE_JOB_RESULT=passed
+              return 130
+            fi
+          done
+        else
+          export MAVEN_EXTRA_ARGS=""
+        fi
       - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER"
       - export AMD_ARCH=.amd64
       - export ARM_ARCH=.arm64
@@ -107,7 +129,7 @@ blocks:
             - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$PACKAGES_URL"
             - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
               -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8
-              $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
+              $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
             - . cache-maven store
             - >-
               for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
@@ -128,6 +150,9 @@ blocks:
     task:
       jobs:
         - name: Deploy AMD confluentinc/kafka-streams-examples ubi8
+          env_vars:
+            - name: DOCKER_IMAGE
+              value: confluentinc/kafka-streams-examples
           commands:
             - export OS_TAG="-ubi8"
             - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/kafka-streams-examples
@@ -156,7 +181,7 @@ blocks:
     task:
       agent:
         machine:
-          type: s1-prod-ubuntu20-04-arm64-1
+          type: s1-prod-ubuntu24-04-arm64-1
       jobs:
         - name: Build & Test ubi8
           commands:
@@ -167,7 +192,7 @@ blocks:
             - ci-tools ci-update-version
             - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
               -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8
-              $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
+              $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
             - . cache-maven store
             - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
       epilogue:
@@ -183,9 +208,12 @@ blocks:
     task:
       agent:
         machine:
-          type: s1-prod-ubuntu20-04-arm64-1
+          type: s1-prod-ubuntu24-04-arm64-1
       jobs:
         - name: Deploy ARM confluentinc/kafka-streams-examples ubi8
+          env_vars:
+            - name: DOCKER_IMAGE
+              value: confluentinc/kafka-streams-examples
           commands:
             - export OS_TAG="-ubi8"
             - export PROD_IMAGE_NAME=${DOCKER_PROD_REGISTRY}confluentinc/kafka-streams-examples
@@ -244,7 +272,7 @@ after_pipeline:
   task:
     agent:
       machine:
-        type: s1-prod-ubuntu20-04-arm64-0
+        type: s1-prod-ubuntu24-04-arm64-0
     jobs:
       - name: Metrics
         commands:
diff --git a/.semaphore/cp_dockerfile_promote.yml b/.semaphore/cp_dockerfile_promote.yml
@@ -7,7 +7,7 @@ version: v1.0
 name: build-test-release
 agent:
   machine:
-    type: s1-prod-ubuntu20-04-amd64-1
+    type: s1-prod-ubuntu24-04-amd64-1
 
 fail_fast:
   cancel:
@@ -46,13 +46,27 @@ global_job_config:
       - docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY
       - export AMD_ARCH=.amd64
       - export ARM_ARCH=.arm64
+      - export COMMUNITY_DOCKER_REPOS=""
+      - |
+        if [[ $SKIP_COMMUNITY == "True" ]]; then
+          for skip_repo in $COMMUNITY_DOCKER_REPOS; do
+            if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
+              echo "Skipping promotion for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
+              export SEMAPHORE_JOB_RESULT=passed
+              return 130
+            fi
+          done
+        fi
 
 blocks:
   - name: Promote AMD
     dependencies: []
     task:
       jobs:
         - name: Promote confluentinc/kafka-streams-examples ubi8 AMD
+          env_vars:
+            - name: DOCKER_IMAGE
+              value: confluentinc/kafka-streams-examples
           commands:
             - export OS_TYPE="ubi8"
             - export DOCKER_REPO="confluentinc/kafka-streams-examples"
@@ -85,6 +99,9 @@ blocks:
     task:
       jobs:
         - name: Promote confluentinc/kafka-streams-examples ubi8 ARM
+          env_vars:
+            - name: DOCKER_IMAGE
+              value: confluentinc/kafka-streams-examples
           commands:
             - export OS_TYPE="ubi8"
             - export DOCKER_REPO="confluentinc/kafka-streams-examples"
@@ -117,6 +134,9 @@ blocks:
     task:
       jobs:
         - name: Create Manifest confluentinc/kafka-streams-examples ubi8
+          env_vars:
+            - name: DOCKER_IMAGE
+              value: confluentinc/kafka-streams-examples
           commands:
             - export OS_TYPE="ubi8"
             - if [[ ! "$OS_TYPE" ]]; then export OS_TAG=""; elif [[ "$OS_TYPE" =~ $PROMOTE_OS_TYPE* ]]; then export OS_TAG="-$OS_TYPE"; fi
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
@@ -7,7 +7,7 @@ version: v1.0
 name: build-test-release
 agent:
   machine:
-    type: s1-prod-ubuntu20-04-amd64-1
+    type: s1-prod-ubuntu24-04-amd64-1
 
 fail_fast:
   cancel:
@@ -71,6 +71,28 @@ global_job_config:
       - export LATEST_TAG=$BRANCH_TAG-latest
       - export DOCKER_UPSTREAM_TAG="$LATEST_TAG"
       - export DOCKER_REPOS="confluentinc/kafka-streams-examples"
+      - export COMMUNITY_DOCKER_REPOS=""
+      - |
+        if [[ $SKIP_COMMUNITY == "True" ]]; then
+          # Filter out community repos from DOCKER_REPOS
+          DOCKER_REPOS=$(comm -23 <(echo "$DOCKER_REPOS" | tr ' ' '\n' | sort) <(echo "$COMMUNITY_DOCKER_REPOS" | tr ' ' '\n' | sort) | tr '\n' ' ' | xargs)
+          export DOCKER_REPOS
+          echo "DOCKER_REPOS after skipping community images - $DOCKER_REPOS"
+
+          # Set Maven arguments for skipping community modules
+          export MAVEN_EXTRA_ARGS=""
+
+          # Check if current DOCKER_IMAGE is in community repos, skip job execution
+          for skip_repo in $COMMUNITY_DOCKER_REPOS; do
+            if [ "$skip_repo" = "$DOCKER_IMAGE" ]; then
+              echo "Skipping build for $DOCKER_IMAGE as it is in COMMUNITY_DOCKER_REPOS list"
+              export SEMAPHORE_JOB_RESULT=passed
+              return 130
+            fi
+          done
+        else
+          export MAVEN_EXTRA_ARGS=""
+        fi
       - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER"
       - export AMD_ARCH=.amd64
       - export ARM_ARCH=.arm64
@@ -99,7 +121,7 @@ blocks:
             - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$PACKAGES_URL"
             - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
               -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8
-              $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
+              $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
             - . cache-maven store
             - >-
               for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES;
@@ -120,7 +142,7 @@ blocks:
     task:
       agent:
         machine:
-          type: s1-prod-ubuntu20-04-arm64-1
+          type: s1-prod-ubuntu24-04-arm64-1
       jobs:
         - name: Build & Test ubi8
           commands:
@@ -131,7 +153,7 @@ blocks:
             - ci-tools ci-update-version
             - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY
               -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8
-              $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true
+              $PACKAGING_BUILD_ARGS -Ddependency.check.skip=true $MAVEN_EXTRA_ARGS
             - . cache-maven store
             - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done
       epilogue:
@@ -144,7 +166,7 @@ after_pipeline:
   task:
     agent:
       machine:
-        type: s1-prod-ubuntu20-04-arm64-0
+        type: s1-prod-ubuntu24-04-arm64-0
     jobs:
       - name: Metrics
         commands:
diff --git a/service.yml b/service.yml
@@ -9,6 +9,8 @@ semaphore:
   enable: true
   pipeline_type: cp-dockerfile
   docker_repos: ['confluentinc/kafka-streams-examples']
+  community_docker_repos: []
+  community_maven_modules: []
   maven_phase: 'package' # streams examples integration-test needs host-based networking, won't work in CI as-is
   maven_skip_deploy: true
   build_arm: true
@@ -40,6 +42,13 @@ semaphore:
         - name: CONFLUENT_DEB_VERSION
           required: false
           default_value: '1'
+        - name: SKIP_COMMUNITY
+          required: false
+          default_value: 'False'
+          description: 'Skip building community docker images'
+          options:
+            - 'True'
+            - 'False'
     - name: cp-dockerfile-promote
       branch: master
       pipeline_file: .semaphore/cp_dockerfile_promote.yml
@@ -58,3 +67,10 @@ semaphore:
           options:
             - 'deb'
             - 'ubi'
+        - name: SKIP_COMMUNITY
+          required: false
+          default_value: 'False'
+          description: 'Skip publishing community docker images'
+          options:
+            - 'True'
+            - 'False'
