Refactor issue creation workflows to dynamically retrieve components …

…from version manifests

Signed-off-by: Sayali Gaikawad <[email protected]>

.github/workflows/create-release-issues.yml

@@ -0,0 +1,91 @@
+---
+name: create-release-issues
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        description: 'Release version'
+        type: string
+      repos:
+        required: true
+        description: 'List of components repositories'
+        type: string
+
+jobs:
+  build-repo-issue-check:
+    outputs:
+      build_repo_issue_exists: ${{ steps.check_if_build_repo_issue_exists.outputs.issues }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Checkout Build repo
+        uses: actions/checkout@v3
+      - name: Check if build repo release issue exists
+        id: check_if_build_repo_issue_exists
+        uses: actions-cool/issues-helper@v3
+        with:
+          actions: 'find-issues'
+          repo: opensearch-project/opensearch-build
+          token: ${{ secrets.TOKS_TOKS }}
+          issue-state: 'open'
+          title-includes: '[RELEASE] Release version ${{ inputs.version }}'
+
+
+  component-release-issue:
+    needs: build-repo-issue-check
+    if: needs.build-repo-issue-check.outputs.build_repo_issue_exists != '[]'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        repos: ${{ fromJson(inputs.repos) }}
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Check if plugin repo release issue exists
+        id: check_if_plugin_repo_issue_exists
+        uses: actions-cool/issues-helper@v3
+        with:
+          actions: 'find-issues'
+          repo: opensearch-project/${{ matrix.repos }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title-includes: '[RELEASE] Release version ${{ inputs.version }}'
+      - name: Checkout Build repo
+        uses: actions/checkout@v3
+      - name: Replace Placeholders
+        if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]'
+        run: |
+          # Read the file contents and replace the placeholders
+          file_path=".github/ISSUE_TEMPLATE/component_release_template.md"
+          RELEASE_VERSION="${{ inputs.version }}"
+          RELEASE_BRANCH=$(echo ${{ inputs.version }} | cut -d. -f1-2)
+          BUILD_REPO_ISSUE_OUTPUT=$(cat <<EOF
+          ${{ needs.build-repo-issue-check.outputs.build_repo_issue_exists.outputs.issues }}
+          EOF
+          )
+          RELEASE_ISSUE_NUMBER=$(echo $BUILD_REPO_ISSUE_OUTPUT | jq -r '.[0].number')
+          RELEASE_ISSUE="https://github.com/opensearch-project/opensearch-build/issues/${RELEASE_ISSUE_NUMBER}"
+          RELEASE_VERSION_X=$(echo "${{ inputs.version }}" | awk -F'.' '{print $1}').x
+          sed -e "s|{{RELEASE_VERSION}}|${RELEASE_VERSION}|g" -e "s|{{RELEASE_ISSUE}}|${RELEASE_ISSUE}|g" -e "s|{{RELEASE_BRANCH}}|${RELEASE_BRANCH}|g" -e "s|{{RELEASE_VERSION_X}}|${RELEASE_VERSION_X}|g" "$file_path" > "$file_path.tmp" && mv "$file_path.tmp" "$file_path"
+      - name: Create component release issue from file
+        if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]'
+        uses: peter-evans/create-issue-from-file@v4
+        with:
+          title: '[RELEASE] Release version ${{ inputs.version }}'
+          content-filepath: ../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md
+          labels: v${{ inputs.version }}
+          token: ${{ steps.github_app_token.outputs.token }}
+          repository: opensearch-project/${{ matrix.repos }}

.github/workflows/os-release-issues.yml

@@ -3,16 +3,6 @@ name: release-issue-os
 
 on:
   workflow_dispatch:
-    inputs:
-      logLevel:
-        description: Log level
-        required: true
-        default: warning
-        type: choice
-        options:
-          - info
-          - warning
-          - debug
   schedule:
     - cron: 0 1 * * *
 
@@ -21,103 +11,59 @@ jobs:
     if: github.repository == 'opensearch-project/opensearch-build'
     runs-on: ubuntu-latest
     outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      versions_matrix: ${{ steps.set-matrix.outputs.versions }}
     steps:
       - uses: actions/checkout@v3
-        with:
-          repository: opensearch-project/opensearch-build
-          ref: main
       - id: set-matrix
-        # produces a list of major versions, e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"]
-        run: echo "::set-output name=matrix::$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')"
-  component-release-issue:
+        # produces a list of major.minor versions only, no patch versions e.g. ["1.4.0","2.10.0","2.6.0","2.7.0","2.8.0","2.9.0","3.0.0"]
+        run: echo "versions=$(ls manifests/**/opensearch*.yml | cut -d'/' -f2 | grep '0$' | grep -v '[0-9]0$' | sort | uniq | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
+
+  list-components-per-version:
     needs: list-manifest-versions
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        entry:
-          - {repo: OpenSearch}
-          - {repo: alerting}
-          - {repo: anomaly-detection}
-          - {repo: asynchronous-search}
-          - {repo: common-utils}
-          - {repo: cross-cluster-replication}
-          - {repo: geospatial}
-          - {repo: index-management}
-          - {repo: job-scheduler}
-          - {repo: k-NN}
-          - {repo: neural-search}
-          - {repo: ml-commons}
-          - {repo: notifications}
-          - {repo: observability}
-          - {repo: performance-analyzer}
-          - {repo: performance-analyzer-rca}
-          - {repo: reporting}
-          - {repo: security}
-          - {repo: security-analytics}
-          - {repo: sql}
-          - {repo: custom-codecs}
-          - {repo: flow-framework}
-          - {repo: skills}
-          - {repo: query-insights}
-          - {repo: opensearch-system-templates}
-        release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }}
+    outputs:
+      version_components_matrix: ${{ steps.get-all-components.outputs.combined_matrix }}
     steps:
-      - name: GitHub App token
-        id: github_app_token
-        uses: tibdex/[email protected]
-        with:
-          app_id: ${{ secrets.APP_ID }}
-          private_key: ${{ secrets.APP_PRIVATE_KEY }}
-          installation_id: 22958780
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Check if build repo release issue exists
-        id: check_if_build_repo_issue_exists
-        uses: actions-cool/issues-helper@v3
-        with:
-          actions: 'find-issues'
-          repo: opensearch-project/opensearch-build
-          token: ${{ steps.github_app_token.outputs.token }}
-          issue-state: 'open'
-          title-includes: '[RELEASE] Release version ${{ matrix.release_version }}'
-      - name: Check out plugin repo
-        uses: actions/checkout@v3
-        with:
-          path: plugin-repo
-          repository: opensearch-project/${{ matrix.entry.repo }}
-      - name: Check if plugin repo release issue exists
-        if: steps.check_if_build_repo_issue_exists.outputs.issues != '[]'
-        id: check_if_plugin_repo_issue_exists
-        uses: actions-cool/issues-helper@v3
-        with:
-          actions: 'find-issues'
-          repo: opensearch-project/${{ matrix.entry.repo }}
-          token: ${{ steps.github_app_token.outputs.token }}
-          title-includes: '[RELEASE] Release version ${{ matrix.release_version }}'
-      - name: Replace Placeholders
-        if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]'
+      - uses: actions/checkout@v3
+
+      - id: get-all-components
         run: |
-          # Read the file contents and replace the placeholders
-          file_path="../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md"
-          RELEASE_VERSION="${{ matrix.release_version }}"
-          RELEASE_BRANCH=$(echo ${{ matrix.release_version }} | cut -d. -f1-2)
-          BUILD_REPO_ISSUE_OUTPUT=$(cat <<EOF
-          ${{ steps.check_if_build_repo_issue_exists.outputs.issues }}
-          EOF
-          )
-          RELEASE_ISSUE_NUMBER=$(echo $BUILD_REPO_ISSUE_OUTPUT | jq -r '.[0].number')
-          RELEASE_ISSUE="https://github.com/opensearch-project/opensearch-build/issues/${RELEASE_ISSUE_NUMBER}"
-          RELEASE_VERSION_X=$(echo "${{ matrix.release_version }}" | awk -F'.' '{print $1}').x
-          sed -e "s|{{RELEASE_VERSION}}|${RELEASE_VERSION}|g" -e "s|{{RELEASE_ISSUE}}|${RELEASE_ISSUE}|g" -e "s|{{RELEASE_BRANCH}}|${RELEASE_BRANCH}|g" -e "s|{{RELEASE_VERSION_X}}|${RELEASE_VERSION_X}|g" "$file_path" > "$file_path.tmp" && mv "$file_path.tmp" "$file_path"
-      - name: Create component release issue from file
-        if: steps.check_if_plugin_repo_issue_exists.outputs.issues == '[]'
-        uses: peter-evans/create-issue-from-file@v4
-        with:
-          title: '[RELEASE] Release version ${{ matrix.release_version }}'
-          content-filepath: ../opensearch-build/.github/ISSUE_TEMPLATE/component_release_template.md
-          labels: |
-            v${{ matrix.release_version }}
-          token: ${{ steps.github_app_token.outputs.token }}
-          repository: opensearch-project/${{ matrix.entry.repo }}
+          versions_array=${{ needs.list-manifest-versions.outputs.versions_matrix }}
+
+          # Remove brackets from the array string
+          versions_array="${versions_array:1:-1}"
+
+          # Split by commas and process each version
+          IFS=',' read -ra VERSIONS <<< "$versions_array"
+
+          # Initialize an array to store all entries
+          declare -a matrix_entries=()
+
+          for version in "${VERSIONS[@]}"; do
+            # Remove quotes and whitespace from version
+            version=$(echo $version | tr -d '"' | xargs)
+
+            # Get components for this version
+            components=$(yq eval '.components[].repository' "manifests/$version/opensearch-$version.yml" | \
+                        sed 's/.*\///;s/\.git$//' | \
+                        jq -R -s -c 'split("\n")[:-1]')
+
+            # Add this version-components pair to the matrix
+            matrix_entries+=("{\"version\":\"$version\",\"components\":$components}")
+          done
+
+          # Combine all entries into the final matrix format
+          matrix_json="{\"include\":[$(IFS=,; echo "${matrix_entries[*]}")]}"
+
+          echo "combined_matrix=$matrix_json" >> $GITHUB_OUTPUT
+
+  trigger-issue-creation-workflow:
+    needs: list-components-per-version
+    strategy:
+      matrix: ${{ fromJson(needs.list-components-per-version.outputs.version_components_matrix) }}
+      fail-fast: false
+    uses: ./.github/workflows/create-release-issues.yml
+    secrets: inherit
+    with:
+      version: ${{ matrix.version }}
+      repos: ${{ toJson(matrix.components) }}