HPCC4J-780 Introduce agent-ready knowledge base #732
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Baremetal Regression Suite | |
| on: | |
| pull_request: | |
| branches: | |
| - "master" | |
| - "candidate-*" | |
| workflow_dispatch: | |
| jobs: | |
| test-against-platform: | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| strategy: | |
| matrix: | |
| security: [enabled, disabled] | |
| steps: | |
| - name: Setup JDK 11 | |
| uses: actions/setup-java@v1 | |
| with: | |
| java-version: 11 | |
| - uses: "actions/setup-python@v2" | |
| with: | |
| python-version: "3.8" | |
| - name: "Install Python dependencies" | |
| run: | | |
| set -xe | |
| python -VV | |
| python -m site | |
| python -m pip install --upgrade pip setuptools wheel | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - name: Extract Latest Tagged Version | |
| id: extract_version | |
| env: | |
| PULL_REQUEST_NUMBER : ${{ github.event.pull_request.number }} | |
| PULL_REQUEST_TITLE : ${{ github.event.pull_request.title }} | |
| PULL_REQUEST_AUTHOR_NAME : ${{ github.event.pull_request.user.login }} | |
| PULL_URL: ${{ github.event.pull_request.html_url }} | |
| COMMENTS_URL: ${{ github.event.pull_request.comments_url }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BRANCH_NAME: ${{ github.base_ref }} | |
| shell: python | |
| run: | | |
| import os | |
| import re | |
| import subprocess | |
| import time | |
| import sys | |
| def extractVersion(versionStr): | |
| parts = versionStr.split('.') | |
| if len(parts) != 3: | |
| print('Invalid version: ' + versionStr) | |
| sys.exit(1) | |
| if parts[2].lower() == 'x': | |
| parts[2] = '0' | |
| major, minor, point = map(int, parts) | |
| return [major, minor, point] | |
| def getTagVersionForCmd(cmd): | |
| versionPattern = re.compile(r'.*hpcc4j_([0-9]+\.[0-9]+\.[0-9]+).*') | |
| # Get latest release version | |
| gitTagProcess = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) | |
| (output, err) = gitTagProcess.communicate() | |
| gitTagProcessStatus = gitTagProcess.wait() | |
| if gitTagProcessStatus != 0: | |
| print('Unable to retrieve latest git tag. With error: ' + str(err)) | |
| return None | |
| latestGitTag = str(output) | |
| versionMatch = versionPattern.match(latestGitTag) | |
| if versionMatch: | |
| return extractVersion(versionMatch.group(1)) | |
| else: | |
| print('Unable to extract version from git tag: ' + latestGitTag) | |
| return None | |
| def getLatestBranchVersion(branchName): | |
| latestVersion = getTagVersionForCmd("git tag --list 'hpcc4j_*-release' --sort=-v:refname | head -n 1") | |
| if latestVersion is None: | |
| print('Unable to find latest release version') | |
| sys.exit(2) | |
| if branchName == 'master': | |
| return [latestVersion[0], latestVersion[1], latestVersion[2]] | |
| else: | |
| # Extract candidate branch major / minor version | |
| candidateBranchPattern = re.compile(r'candidate-([0-9]+\.[0-9]+\.([0-9]+|x)).*') | |
| branchVersionMatch = candidateBranchPattern.match(branchName) | |
| if branchVersionMatch is None: | |
| print('Unable to extract version from branch name: ' + branchName) | |
| sys.exit(3) | |
| branchVersion = extractVersion(branchVersionMatch.group(1)) | |
| # Get latest release in branch | |
| findLatestBranchVer = "git tag --list 'hpcc4j_" + str(branchVersion[0]) + "." + str(branchVersion[1]) + "*-release' --sort=-v:refname | head -n 1" | |
| latestBranchVersion = getTagVersionForCmd(findLatestBranchVer) | |
| if latestBranchVersion is None: | |
| print('No release found for branch: ' + branchName + ' using latest release: ' + str(latestVersion)) | |
| return latestVersion | |
| else: | |
| return latestBranchVersion | |
| branch_name = os.environ['BRANCH_NAME'] | |
| latestVersion = getLatestBranchVersion(branch_name) | |
| if latestVersion[2] == 0: | |
| print('Latest version is a new minor. Setting previous version to latest version') | |
| previousVersion = latestVersion | |
| else: | |
| previousVersion = [latestVersion[0], latestVersion[1], latestVersion[2] - 2] | |
| previousVersionStr = ".".join(map(str, previousVersion)) | |
| previousVersionURL = 'https://github.com/hpcc-systems/HPCC-Platform/releases/download/community_' + previousVersionStr \ | |
| + '-1/hpccsystems-platform-community_' + previousVersionStr + '-1jammy_amd64_withsymbols.deb' | |
| latestVersionStr = ".".join(map(str, latestVersion)) | |
| latestVersionURL = 'https://github.com/hpcc-systems/HPCC-Platform/releases/download/community_' + latestVersionStr \ | |
| + '-1/hpccsystems-platform-community_' + latestVersionStr + '-1jammy_amd64_withsymbols.deb' | |
| github_output_path = os.getenv('GITHUB_OUTPUT') | |
| if github_output_path: | |
| with open(github_output_path, 'a') as output_file: | |
| output_file.write(f"previousVersion={previousVersionStr}\n") | |
| output_file.write(f"previousVersionURL={previousVersionURL}\n") | |
| output_file.write(f"latestVersion={latestVersionStr}\n") | |
| output_file.write(f"latestVersionURL={latestVersionURL}\n") | |
| else: | |
| print('GITHUB_OUTPUT environment variable is not set.') | |
| - name: Install latest version | |
| run: | | |
| if wget -q --spider ${{ steps.extract_version.outputs.latestVersionURL }}; then | |
| wget -q ${{ steps.extract_version.outputs.latestVersionURL }} | |
| elif wget -q --spider ${{ steps.extract_version.outputs.previousVersionURL }}; then | |
| wget -q ${{ steps.extract_version.outputs.previousVersionURL }} | |
| else | |
| echo "Unable to find HPCC version to install" | |
| exit 1 | |
| fi | |
| sudo apt-get update | |
| sudo apt-get install -y expect | |
| sudo dpkg -i hpccsystems-platform-community_*.deb | |
| sudo apt-get -f install -y | |
| - name: Setup Certificates | |
| run: | | |
| sudo /opt/HPCCSystems/etc/init.d/setupPKI | |
| - name: Write htpasswd file | |
| if: matrix.security == 'enabled' | |
| shell: bash | |
| run: | | |
| # Generate random credentials | |
| DUMMY_USER=$(openssl rand -hex 8) | |
| DUMMY_PASS=$(openssl rand -hex 16) | |
| JUNIT_USER=$(openssl rand -hex 4)@$(openssl rand -hex 4) | |
| # Export credentials as environment variables | |
| echo "DUMMY_USER=$DUMMY_USER" >> $GITHUB_ENV | |
| echo "DUMMY_PASS=$DUMMY_PASS" >> $GITHUB_ENV | |
| echo "JUNIT_USER=$JUNIT_USER" >> $GITHUB_ENV | |
| # Create .htpasswd file with user:password | |
| sudo apt-get install -y apache2-utils | |
| # Add dummy user and password to .htpasswd file | |
| sudo htpasswd -c -b /etc/HPCCSystems/.htpasswd $DUMMY_USER $DUMMY_PASS | |
| # Add test user and password to .htpasswd file | |
| sudo htpasswd -b /etc/HPCCSystems/.htpasswd $JUNIT_USER $DUMMY_PASS | |
| - name: Add htpasswdsecmgr to environment.xml | |
| if: matrix.security == 'enabled' | |
| shell: python | |
| run: | | |
| import xml.etree.ElementTree as ET | |
| # Parse the environment.xml file | |
| env_file = '/etc/HPCCSystems/environment.xml' | |
| tree = ET.parse(env_file) | |
| root = tree.getroot() | |
| # 1. Add BuildSet under Environment/Programs/Build | |
| programs = root.find('./Programs') | |
| build = programs.find('./Build') | |
| # Create the BuildSet element with attributes | |
| buildset = ET.SubElement(build, 'BuildSet') | |
| buildset.set('deployable', 'no') | |
| buildset.set('installSet', 'deploy_map.xml') | |
| buildset.set('name', 'htpasswdsecmgr') | |
| buildset.set('path', 'componentfiles/htpasswdsecmgr') | |
| buildset.set('processName', 'HtpasswdSecurityManager') | |
| buildset.set('schema', 'htpasswd_secmgr.xsd') | |
| # 2. Update Authentication method attribute | |
| software = root.find('./Software') | |
| esp_processes = software.findall('./EspProcess') | |
| for esp in esp_processes: | |
| auth = esp.find('./Authentication') | |
| if auth is not None: | |
| auth.set('method', 'secmgrPlugin') | |
| # 3. Update only EspBinding with name "myespsmc" type attribute | |
| for esp in esp_processes: | |
| bindings = esp.findall('./EspBinding') | |
| for binding in bindings: | |
| if binding.get('name') == 'myespsmc': | |
| binding.set('type', 'htpasswdsecmgr') | |
| # 4. Add HtpasswdSecurityManager node under Software | |
| htpasswd_mgr = ET.SubElement(software, 'HtpasswdSecurityManager') | |
| htpasswd_mgr.set('build', '_') | |
| htpasswd_mgr.set('buildSet', 'htpasswdsecmgr') | |
| htpasswd_mgr.set('htpasswdFile', '/etc/HPCCSystems/.htpasswd') | |
| htpasswd_mgr.set('instanceFactoryName', 'createInstance') | |
| htpasswd_mgr.set('libName', 'libhtpasswdSecurity.so') | |
| htpasswd_mgr.set('name', 'htpasswdsecmgr') | |
| htpasswd_mgr.set('type', 'SecurityManager') | |
| # Write changes back to the file | |
| tree.write('tmp_environment.xml') | |
| print("Environment.xml updated successfully for security configuration") | |
| - name: Replace environment.xml | |
| if: matrix.security == 'enabled' | |
| run: | | |
| sudo mv /etc/HPCCSystems/environment.xml /etc/HPCCSystems/environment.xml.bak | |
| sudo mv tmp_environment.xml /etc/HPCCSystems/environment.xml | |
| sudo chown hpcc:hpcc /etc/HPCCSystems/environment.xml | |
| - name: Start HPCC-Platforma | |
| shell: "bash" | |
| run: | | |
| export LANG="en_US.UTF-8" | |
| sudo update-locale | |
| sudo /etc/init.d/hpcc-init start | |
| - name: Add Host File Entries | |
| run: | | |
| sudo -- sh -c -e "echo '127.0.0.1 eclwatch.default' >> /etc/hosts"; | |
| sudo -- sh -c -e "echo '127.0.0.1 rowservice.default' >> /etc/hosts"; | |
| sudo -- sh -c -e "echo '127.0.0.1 sql2ecl.default' >> /etc/hosts"; | |
| # speed things up with caching from https://docs.github.com/en/actions/guides/building-and-testing-java-with-maven | |
| - name: Cache Maven packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.m2 | |
| key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: ${{ runner.os }}-m2 | |
| - name: Build with Maven (Secure) | |
| if: matrix.security == 'enabled' | |
| run: mvn -B --activate-profiles jenkins-on-demand,spark33 -Dmaven.gpg.skip=true -Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=true -Dhpccconn=http://eclwatch.default:8010 -Dwssqlconn=http://sql2ecl.default:8510 -DHPCC30117=open -Dhpccuser=${{ env.DUMMY_USER}} -Dhpccpass=${{ env.DUMMY_PASS }} -DencodedUserName=${{ env.JUNIT_USER }} install | |
| - name: Build with Maven (Insecure) | |
| if: matrix.security == 'disabled' | |
| run: mvn -B --activate-profiles jenkins-on-demand,spark33 -Dmaven.gpg.skip=true -Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=true -Dhpccconn=http://eclwatch.default:8010 -Dwssqlconn=http://sql2ecl.default:8510 -DHPCC30117=open install | |
| - name: Process Errors | |
| shell: python | |
| run: | | |
| import os | |
| import csv | |
| import textwrap | |
| import json | |
| import sys | |
| hadErrors = False | |
| failedTestPaths = ["./wsclient/FailedTests.csv", "./dfsclient/FailedTests.csv", "./spark-hpcc/FailedTests.csv"] | |
| for file_path in failedTestPaths: | |
| if os.path.exists(file_path): | |
| with open(file_path, 'r') as file: | |
| csv_reader = csv.reader(file) | |
| for row in csv_reader: | |
| # If row is empty skip | |
| if not row: | |
| continue | |
| hadErrors = True | |
| # Each row in the CSV file is a failed test with: TestClass,Test,Error | |
| if len(row) == 3: | |
| print(f"::error file={row[0]}.{row[1]} title={row[2]}") | |
| else: | |
| print(f"::error file={file_path} title=Invalid error row: {row}") | |
| else: | |
| print(f"FailedTests.csv does not exist at {file_path}") | |
| if hadErrors: | |
| sys.exit(1) |