HPCC4J-780 Introduce agent-ready knowledge base #726
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: K8s Regression Suite | |
| on: | |
| pull_request: | |
| branches: | |
| - "master" | |
| - "candidate-*" | |
| workflow_dispatch: | |
| jobs: | |
| parse-pr-body: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| repository: ${{ steps.parse-pr-body.outputs.repository }} | |
| branch: ${{ steps.parse-pr-body.outputs.branch }} | |
| has-custom-platform: ${{ steps.parse-pr-body.outputs.has-custom-platform == 'true' }} | |
| steps: | |
| - name: Parse PR Body | |
| id: parse-pr-body | |
| uses: actions/github-script@v7 | |
| env: | |
| PROJECT_CONFIG: ${{ vars.PROJECT_CONFIG }} | |
| with: | |
| script: | | |
| if (context.eventName === 'workflow_dispatch') { | |
| core.info('Manual trigger - using default platform'); | |
| core.setOutput("repository", ""); | |
| core.setOutput("branch", ""); | |
| core.setOutput("has-custom-platform", false); | |
| return; | |
| } | |
| const body = context.payload.pull_request?.body || ""; | |
| console.log(`PR body: ${body}`); | |
| let repository = ""; | |
| let branch = ""; | |
| // Parse the project config from environment variable | |
| // This should be a JSON object containing the "allowedRepositories" field, | |
| // which is an array of fully qualified repository names IE: [ "hpcc-systems/hpcc-platform", "..."] | |
| let projectConfig = {}; | |
| try { | |
| const configStr = process.env.PROJECT_CONFIG || "{}"; | |
| projectConfig = JSON.parse(configStr); | |
| console.log(`Parsed project config:`, projectConfig); | |
| } catch (error) { | |
| console.log(`Failed to parse PROJECT_CONFIG: ${error.message}`); | |
| projectConfig = {}; | |
| } | |
| // Get allowed repositories from project config | |
| const allowedRepos = projectConfig.allowedRepositories || []; | |
| if (allowedRepos.length > 0) { | |
| console.log(`Allowed repositories: ${allowedRepos.join(", ")}`); | |
| } else { | |
| console.log(`No fully qualified repository whitelist configured - no external repositories allowed`); | |
| allowedRepos.push("hpcc-systems/hpcc-platform"); // Default to HPCC-Platform if no whitelist is configured | |
| } | |
| const repoMatch = body.match(/repository:\s*([^\n]+)/); | |
| if (repoMatch) repository = repoMatch[1].trim().toLowerCase(); | |
| console.log(`Parsed repository: ${repository}`); | |
| const branchMatch = body.match(/branch:\s*([^\n]+)/); | |
| if (branchMatch) branch = branchMatch[1].trim().toLowerCase(); | |
| console.log(`Parsed branch: ${branch}`); | |
| // Lowercase allowed repository names to ensure case-insensitive matching | |
| const lowerCaseAllowedRepos = allowedRepos.map(repo => repo.toLowerCase()); | |
| console.log(`Allowed repositories: ${lowerCaseAllowedRepos.join(", ")}`); | |
| if (repository && !lowerCaseAllowedRepos.includes(repository)) { | |
| console.log(`Repository ${repository} is not allowed. Using default platform.`); | |
| repository = ""; | |
| branch = ""; | |
| } | |
| // Sanitize branch name, limit to branches with the following: [a-zA-Z0-9._-/] | |
| // If the branch contains any invalid characters, it will be set to an empty string. | |
| const branchRegex = /^[a-zA-Z0-9._\-\/]+$/; | |
| if (branch && !branchRegex.test(branch)) { | |
| console.log(`Branch name contains invalid characters. Using default platform.`); | |
| branch = ""; | |
| } | |
| const hasCustomPlatform = repository != "" && branch != ""; | |
| if (!hasCustomPlatform) { | |
| console.log(`No custom repository/branch specified. Using default platform.`); | |
| } else { | |
| console.log(`Custom platform found: ${repository}@${branch}`); | |
| } | |
| core.setOutput("repository", repository); | |
| core.setOutput("branch", branch); | |
| core.setOutput("has-custom-platform", hasCustomPlatform); | |
| build-docker: | |
| name: build-docker-ubuntu-22.04 | |
| needs: parse-pr-body | |
| uses: ./.github/workflows/build-docker.yml | |
| with: | |
| os: ubuntu-22.04 | |
| should-build: ${{ needs.parse-pr-body.outputs.has-custom-platform == 'true' }} | |
| repository: ${{ needs.parse-pr-body.outputs.repository }} | |
| branch: ${{ needs.parse-pr-body.outputs.branch }} | |
| upload-package: true | |
| containerized: true | |
| asset-name: docker-ubuntu-22_04-containerized | |
| secrets: inherit | |
| test-against-platform: | |
| runs-on: ubuntu-latest | |
| needs: [parse-pr-body, build-docker] | |
| steps: | |
| - name: Setup JDK 11 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: 11 | |
| - name: Install K8s | |
| uses: balchua/[email protected] | |
| with: | |
| channel: '1.26/stable' | |
| devMode: 'true' | |
| addons: '["dns", "rbac", "hostpath-storage", "registry"]' | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: Checkout HPCC-Platform | |
| if: ${{ needs.parse-pr-body.outputs.has-custom-platform == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| path: ./HPCC-Platform | |
| repository: ${{ needs.parse-pr-body.outputs.repository }} | |
| ref: ${{ needs.parse-pr-body.outputs.branch }} | |
| - name: Build HPCC Docker Image | |
| if: ${{ needs.parse-pr-body.outputs.has-custom-platform == 'true' }} | |
| uses: ./.github/actions/build-k8s | |
| with: | |
| platform-folder: ./HPCC-Platform | |
| - name: Deploy HPCC on K8s | |
| uses: ./.github/actions/deploy-hpcc-k8s | |
| with: | |
| use-local-image: ${{ needs.parse-pr-body.outputs.has-custom-platform == 'true' }} | |
| platform-folder: ./HPCC-Platform | |
| # speed things up with caching from https://docs.github.com/en/actions/guides/building-and-testing-java-with-maven | |
| - name: Cache Maven packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2 | |
| - name: Build with Maven | |
| run: mvn -B --activate-profiles jenkins-on-demand,spark33 -Dmaven.gpg.skip=true -Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=true -Dhpccconn=https://eclwatch.default:8010 -Dwssqlconn=https://sql2ecl.default:8510 -DHPCC30117=open install | |
| - name: Rowservice Logs | |
| run: | | |
| echo "DFSClient tests failed - collecting rowservice pod logs" | |
| ROWSERVICE_POD=$(kubectl get pods -l server=rowservice -o jsonpath='{.items[0].metadata.name}') | |
| echo "Rowservice pod: $ROWSERVICE_POD" | |
| kubectl logs $ROWSERVICE_POD > rowservice.log | |
| - name: ECLWatch Logs | |
| run: | | |
| ESP_POD=$(kubectl get pods -l app=eclwatch -o jsonpath='{.items[0].metadata.name}') | |
| echo "ESP pod: $ESP_POD" | |
| kubectl logs $ESP_POD > eclwatch.log | |
| - name: SQL2ECL Logs | |
| run: | | |
| SQL_TO_ECL_POD=$(kubectl get pods -l app=sql2ecl -o jsonpath='{.items[0].metadata.name}') | |
| echo "SQL to ECL pod: $SQL_TO_ECL_POD" | |
| kubectl logs $SQL_TO_ECL_POD > sql2ecl.log | |
| - name: Upload Logs and Test Results | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: k8s-test-results-${{ github.run_number }} | |
| path: | | |
| rowservice.log | |
| eclwatch.log | |
| sql2ecl.log | |
| **/FailedTests.csv | |
| **/*.log | |
| retention-days: 30 | |
| - name: Process Errors | |
| shell: python | |
| run: | | |
| import os | |
| import csv | |
| import textwrap | |
| import json | |
| import sys | |
| hadErrors = False | |
| failedTestPaths = ["./wsclient/FailedTests.csv", "./dfsclient/FailedTests.csv", "./spark-hpcc/FailedTests.csv"] | |
| for file_path in failedTestPaths: | |
| if os.path.exists(file_path): | |
| with open(file_path, 'r') as file: | |
| csv_reader = csv.reader(file) | |
| for row in csv_reader: | |
| # If row is empty skip | |
| if not row: | |
| continue | |
| hadErrors = True | |
| # Each row in the CSV file is a failed test with: TestClass,Test,Error | |
| if len(row) == 3: | |
| print(f"::error file={row[0]}.{row[1]} title={row[2]}") | |
| else: | |
| print(f"::error file={file_path} title=Invalid error row: {row}") | |
| else: | |
| print(f"FailedTests.csv does not exist at {file_path}") | |
| if hadErrors: | |
| sys.exit(1) |