Merge remote-tracking branch 'origin/candidate-9.4.x' into candidate-9.6.x

Signed-off-by: Gordon Smith <[email protected]>

# Conflicts:
#	commons-hpcc/pom.xml
#	dfsclient/pom.xml
#	pom.xml
#	wsclient/pom.xml

Author: GordonSmith

.github/workflows/Jirabot.yml

@@ -14,7 +14,7 @@ jobs:
   jirabot:
     runs-on: ubuntu-20.04
     steps:
-      - uses: "actions/setup-python@v2"
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "3.8"
       - name: "Install dependencies"
@@ -23,9 +23,8 @@ jobs:
           python -VV
           python -m site
           python -m pip install --upgrade pip setuptools wheel
-          python -m pip install --upgrade jira
+          python -m pip install --upgrade atlassian-python-api
           python -m pip --version
-          python -m pip freeze | grep jira
       - name: "Run"
         env:
           JIRABOT_USERNAME : ${{ secrets.JIRABOT_USERNAME }}
@@ -43,49 +42,61 @@ jobs:
         run: |
             import os
             import re
+            import time
+            import sys
             import json
-            from jira.client import JIRA
+            from atlassian.jira import Jira
 
-            def updateIssue(jira, issue, prAuthorEmail : str, transitionMap: dict, propertyMap: dict, pull_url: str) -> str:
+            def updateIssue(jira, issue, prAuthor : str, transitionMap: dict, propertyMap: dict, pull_url: str) -> str:
                 result = ''
 
-                statusName = str(issue.fields.status)
+                issueName = issue['key']
+                issueFields = issue['fields']
+
+                statusName = str(issueFields['status']['name'])
                 transition = transitionMap.get(statusName, None)
 
                 if transition == None:
                     print('Error: Unable to find transition for status: ' + statusName)
                 elif transition != '':
                     try:
-                        jira.transition_issue(issue, transition)
+                        jira.issue_transition(issueName, transition)
                         result += 'Workflow Transition: ' + transition + '\n'
                     except Exception as error:
-                        transitions = jira.transitions(issue)
+                        transitions = jira.get_issue_transitions(issueName)
                         result += 'Error: Transition: "' + transition + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n'
 
                 prFieldName = propertyMap.get('pullRequestFieldName', 'customfield_10010')
-                try:
-                    currentPR = getattr(issue.fields, prFieldName)
-                except:
+
+                if prFieldName in issueFields:
+                    currentPR = issueFields[prFieldName]
+                else:
+                    print('Error: Unable to find pull request field with field name: ' + prFieldName)
                     currentPR = None
-                    print('Error: Unable to get current pull request with field name: ' + prFieldName)
 
                 if currentPR is None:
-                    issue.update(fields={prFieldName: pull_url})
+                    jira.update_issue_field(issueName, {prFieldName: pull_url})
                     result += 'Updated PR\n'
                 elif currentPR is not None and currentPR != pull_url:
                     result += 'Additional PR: ' + pull_url + '\n'
 
-                if prAuthorEmail:
-                    if issue.fields.assignee is None:
-                        jira.assign_issue(issue, prAuthorEmail)
-                        result += 'Assigning user: ' + prAuthorEmail + '\n'
+                if prAuthor:
+                    assignee = issueFields['assignee']
+                    if assignee is None:
+                        assigneeId = ''
+                        assigneeEmail = ''
                     else:
-                        assigneeEmail = None
-                        if issue.fields.assignee:
-                            assigneeEmail = issue.fields.assignee.emailAddress
-                        if assigneeEmail is None or assigneeEmail.lower() != assigneeEmail.lower():
-                            result += 'Changing assignee from: ' + assigneeEmail + ' to: ' + prAuthorEmail + '\n'
-                            jira.assign_issue(issue, prAuthorEmail)
+                        assigneeId = assignee['accountId']
+                        assigneeEmail = assignee["emailAddress"]
+
+                    prAuthorId = prAuthor["accountId"]
+                    prAuthorEmail = prAuthor["emailAddress"]
+                    if assigneeId is None or assigneeId == '':
+                        jira.assign_issue(issueName, prAuthorId)
+                        result += 'Assigning user: ' + prAuthorEmail + '\n'
+                    elif assigneeId != prAuthorId:
+                        result += 'Changing assignee from: ' + assigneeEmail + ' to: ' + prAuthorEmail + '\n'
+                        jira.assign_issue(issueName, prAuthorId)
 
                 return result
 
@@ -114,34 +125,20 @@ jobs:
                     prAuthor = userDict.get(prAuthor)
                     print('Mapped Github user to Jira user: ' + prAuthor)
 
-                options = {
-                    'server': jira_url
-                }
-
-                jira = JIRA(options=options, basic_auth=(jirabot_user, jirabot_pass))
+                jira = Jira(url=jira_url, username= jirabot_user, password= jirabot_pass, cloud=True)
 
-                # Need to change how we find users for Jira Cloud, unfortunately the API doesn't provide this information.
-                # At the moment checking if the URL contains atlassian.net appears to be the easiest way to determine if it's Jira Cloud.
-                isJiraCloud = False
-                if jira_url.find('atlassian.net') > 0:
-                    isJiraCloud = True
-
-                if isJiraCloud:
-                    res = jira.search_users(query=prAuthor)
-                    if res and len(res) > 0:
-                        jiraUser = res[0]
+                jiraUser = None
+                userSearchResults = jira.user_find_by_user_string(query=prAuthor)
+                if userSearchResults and len(userSearchResults) > 0:
+                    jiraUser = userSearchResults[0]
                 else:
-                    jiraUser = jira.user(prAuthor)
-
-                jiraUserEmail = None
-                if jiraUser is None:
                     print('Error: Unable to find Jira user: ' + prAuthor + ' continuing without assigning')
-                else:
-                    jiraUserEmail = jiraUser.emailAddress
 
-                print("Jira User Email:" + str(jiraUserEmail))
+                if not jira.issue_exists(issue_name):
+                    sys.exit('Error: Unable to find Jira issue: ' + issue_name)
+                else:
+                    issue = jira.issue(issue_name)
 
-                issue = jira.issue(issue_name)
                 result = 'Jirabot Action Result:\n'
 
                 transitionMap = json.loads(os.environ['JIRA_ISSUE_TRANSITION_MAP'])
@@ -154,10 +151,18 @@ jobs:
                     print('Error: JIRA_ISSUE_PROPERTY_MAP is not a valid JSON object, ignoring.')
                     jiraIssuePropertyMap = {}
 
-                result += updateIssue(jira, issue, jiraUserEmail, transitionMap, jiraIssuePropertyMap, pull_url)
-                jira.add_comment(issue, result)
+                result += updateIssue(jira, issue, jiraUser, transitionMap, jiraIssuePropertyMap, pull_url)
+                jira.issue_add_comment(issue_name, result)
+
+                result = 'Jira Issue: ' + jira_url + '/browse/' + issue_name + '\n\n' + result
+
+                # Escape the result for JSON
+                result = json.dumps(result)
+
+                curlCommand = 'curl -X POST %s -H "Content-Type: application/json" -H "Authorization: token %s" --data \'{ "body": %s }\'' % ( comments_url, github_token, result )
+                os.system(curlCommand)
             else:
                 print('Unable to find Jira issue name in title')
 
             print(result)
-        shell: python
+        shell: python

.github/workflows/baremetal-regression-suite.yml

@@ -73,7 +73,7 @@ jobs:
 
             if gitTagProcessStatus != 0:
                 print('Unable to retrieve latest git tag. With error: ' + str(err))
-                sys.exit(1)
+                return None
 
             latestGitTag = str(output)
 
@@ -82,11 +82,14 @@ jobs:
                 return extractVersion(versionMatch.group(1))
             else:
                 print('Unable to extract version from git tag: ' + latestGitTag)
-                sys.exit(2)
+                return None
 
         def getLatestBranchVersion(branchName):
 
             latestVersion = getTagVersionForCmd("git tag --list 'hpcc4j_*-release' --sort=-v:refname | head -n 1")
+            if latestVersion is None:
+                print('Unable to find latest release version')
+                sys.exit(2)
 
             if branchName == 'master':
                 return [latestVersion[0], latestVersion[1], latestVersion[2]]
@@ -102,7 +105,12 @@ jobs:
 
                 # Get latest release in branch
                 findLatestBranchVer = "git tag --list 'hpcc4j_" + str(branchVersion[0]) + "." + str(branchVersion[1]) + "*-release' --sort=-v:refname | head -n 1"
-                return getTagVersionForCmd(findLatestBranchVer)
+                latestBranchVersion = getTagVersionForCmd(findLatestBranchVer)
+                if latestBranchVersion is None:
+                    print('No release found for branch: ' + branchName + ' using latest release: ' + str(latestVersion))
+                    return latestVersion
+                else:
+                    return latestBranchVersion
 
         branch_name = os.environ['BRANCH_NAME']
 

dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java

@@ -92,25 +92,43 @@ public void readWithForcedTimeoutTest() throws Exception
     public void nullCharTests() throws Exception
     {
         // Unicode
+        boolean unicodePassed = true;
         {
             FieldDef recordDef = null;
             {
                 FieldDef[] fieldDefs = new FieldDef[2];
                 fieldDefs[0] = new FieldDef("uni", FieldType.STRING, "STRING", 100, false, false, HpccSrcType.UTF16LE, new FieldDef[0]);
-                fieldDefs[1] = new FieldDef("fixedUni", FieldType.STRING, "STRING", 100, true, false, HpccSrcType.UTF16LE, new FieldDef[0]);
+                fieldDefs[1] = new FieldDef("fixedUni", FieldType.STRING, "STRING", 200, true, false, HpccSrcType.UTF16LE, new FieldDef[0]);
 
                 recordDef = new FieldDef("RootRecord", FieldType.RECORD, "rec", 4, false, false, HpccSrcType.LITTLE_ENDIAN, fieldDefs);
             }
 
             List<HPCCRecord> records = new ArrayList<HPCCRecord>();
             int maxUTF16BMPChar = Character.MAX_CODE_POINT;
-            for (int i = 0; i < maxUTF16BMPChar; i++) {
+            for (int i = 0; i < maxUTF16BMPChar; i++)
+            {
+
                 String strMidEOS = "";
-                for (int j = 0; j < 98; j++, i++) {
-                    if (j == 50) {
+                for (int j = 0; j < 98; j++, i++)
+                {
+                    if (!Character.isValidCodePoint(i) || !Character.isDefined(i))
+                    {
+                        continue;
+                    }
+
+                    char[] chars = Character.toChars(i);
+                    if (Character.isSurrogate(chars[0]))
+                    {
+                        continue;
+                    }
+
+                    if (j == 50 && strMidEOS.length() > 0)
+                    {
                         strMidEOS += "\0";
                     }
-                    strMidEOS += Character.toString((char) i);
+
+                    String charStr = new String(chars);
+                    strMidEOS += charStr;
                 }
 
                 Object[] fields = {strMidEOS, strMidEOS};
@@ -123,17 +141,20 @@ public void nullCharTests() throws Exception
             HPCCFile file = new HPCCFile(fileName, connString , hpccUser, hpccPass);
             List<HPCCRecord> readRecords = readFile(file, 10000, false, false, BinaryRecordReader.TRIM_STRINGS);
 
-            for (int i = 0; i < records.size(); i++) {
+            for (int i = 0; i < records.size(); i++)
+            {
                 HPCCRecord record = records.get(i);
                 HPCCRecord readRecord = readRecords.get(i);
                 if (readRecord.equals(record) == false)
                 {
                     System.out.println("Record: " + i + " did not match\n" + record + "\n" + readRecord);
+                    unicodePassed = false;
                 }
             }
         }
 
         // SBC / ASCII
+        boolean sbcPassed = true;
         {
             FieldDef recordDef = null;
             {
@@ -145,13 +166,16 @@ public void nullCharTests() throws Exception
             }
 
             List<HPCCRecord> records = new ArrayList<HPCCRecord>();
-            for (int i = 0; i < 255; i++) {
+            for (int i = 0; i < 255; i++)
+            {
                 String strMidEOS = "";
-                for (int j = 0; j < 9; j++, i++) {
-                    if (j == 5) {
+                for (int j = 0; j < 9; j++, i++)
+                {
+                    if (j == 5)
+                    {
                         strMidEOS += "\0";
                     }
-                    strMidEOS += Character.toString((char) i);
+                    strMidEOS += new String(Character.toChars(j));
                 }
 
                 Object[] fields = {strMidEOS, strMidEOS};
@@ -164,15 +188,20 @@ public void nullCharTests() throws Exception
             HPCCFile file = new HPCCFile(fileName, connString , hpccUser, hpccPass);
             List<HPCCRecord> readRecords = readFile(file, 10000, false, false, BinaryRecordReader.TRIM_STRINGS);
 
-            for (int i = 0; i < records.size(); i++) {
+            for (int i = 0; i < records.size(); i++)
+            {
                 HPCCRecord record = records.get(i);
                 HPCCRecord readRecord = readRecords.get(i);
                 if (readRecord.equals(record) == false)
                 {
                     System.out.println("Record: " + i + " did not match\n" + record + "\n" + readRecord);
+                    sbcPassed = false;
                 }
             }
         }
+
+        assertTrue("Unicode EOS character test failed. See mismatches above.", unicodePassed);
+        assertTrue("Single byte EOS character test failed. See mismatches above.", sbcPassed);
     }
 
     @Test

wsclient/src/main/java/org/hpccsystems/ws/client/antlr/EclRecord.g4

@@ -77,6 +77,8 @@ opts:
 opt:
     maxlength
     | maxcount
+    | setdefaultvalall
+    | setdefaultval
     | defaultval
     | xpath
     | xmldefaultval
@@ -93,7 +95,7 @@ maxcount:
 ;
 
 defaultval:
-    'DEFAULT' OPAREN STRING CPAREN
+    ('DEFAULT' OPAREN STRING CPAREN)
 ;
 
 xpath:
@@ -104,6 +106,13 @@ xmldefaultval:
     'XMLDEFAULT' OPAREN STRING CPAREN
 ;
 
+setdefaultval:
+    'DEFAULT' OPAREN SETSTRING CPAREN
+;
+setdefaultvalall:
+    'DEFAULT' OPAREN 'ALL' CPAREN
+;
+
 annotation_name : ATOKEN;
 annotation_param : (TOKEN|UTOKEN);
 annotation_arguments : annotation_param (COMMA annotation_param)*;
@@ -114,6 +123,8 @@ comment:
 	( '/*' annotation?  (COMMA annotation)* .*? (.*?'*/' | '*/'))
 ;
 
+OSQUARE            : '[';
+CSQUARE            : ']';
 OPAREN             : '(';
 CPAREN             : ')';
 OCURLY             : '{';
@@ -130,9 +141,11 @@ DATASET_SYM            : 'DATASET';
 WS : [ \t\r\n] -> skip;
 INT     : [0-9]+ ;
 fragment ESCAPED_QUOTE : '\\\'';
+SETSTRING :    '[\'' ( ESCAPED_QUOTE | SETTOKEN | ('\'') | ~(']'))* '\']';
 STRING :   '\'' ( ESCAPED_QUOTE | ~('\'') )* '\'';
 ATOKEN: [@][a-zA-Z0-9_-]+[a-zA-Z0-9_];
 TOKEN :  ~[_\r\n\t; (),:={}-]~[\r\n \t;(),:={}-]* ;
 UTOKEN: [_]+[a-zA-Z0-9_-]+[a-zA-Z0-9_];
 ECL_NUMBERED_TYPE: TOKEN INT?;
+SETTOKEN: [a-zA-Z0-9,_-];