HPCC4J-697 Update Spark Tests for HPCC4j Repo (#801)

Signed-off-by: James McMullan [email protected]
hpcc-systems · Mar 11, 2025 · 0e78527 · 0e78527
1 parent d790950
commit 0e78527
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 17 deletions.
diff --git a/.github/workflows/baremetal-regression-suite.yml b/.github/workflows/baremetal-regression-suite.yml
@@ -182,4 +182,4 @@ jobs:
         restore-keys: ${{ runner.os }}-m2
 
     - name: Build with Maven
-      run: mvn -B --activate-profiles jenkins-on-demand -Dmaven.gpg.skip=true -Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=false -Dhpccconn=http://eclwatch.default:8010 -Dwssqlconn=http://sql2ecl.default:8510 -DHPCC30117=open install
+      run: mvn -B --activate-profiles jenkins-on-demand,spark33 -Dmaven.gpg.skip=true -Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=false -Dhpccconn=http://eclwatch.default:8010 -Dwssqlconn=http://sql2ecl.default:8510 -DHPCC30117=open install
diff --git a/.github/workflows/k8s-regression-suite.yml b/.github/workflows/k8s-regression-suite.yml
@@ -144,4 +144,4 @@ jobs:
         restore-keys: ${{ runner.os }}-m2
 
     - name: Build with Maven
-      run: mvn -B --activate-profiles jenkins-on-demand -Dmaven.gpg.skip=true -Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=false -Dhpccconn=https://eclwatch.default:8010 -Dwssqlconn=https://sql2ecl.default:8510 -DHPCC30117=open  install
+      run: mvn -B --activate-profiles jenkins-on-demand,spark33 -Dmaven.gpg.skip=true -Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=false -Dhpccconn=https://eclwatch.default:8010 -Dwssqlconn=https://sql2ecl.default:8510 -DHPCC30117=open  install
diff --git a/pom.xml b/pom.xml
@@ -163,16 +163,6 @@
       <artifactId>json</artifactId>
       <version>${json.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.antlr</groupId>
-      <artifactId>antlr4</artifactId>
-      <version>${antlr.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.antlr</groupId>
-      <artifactId>antlr4-runtime</artifactId>
-      <version>${antlr.version}</version>
-    </dependency>
     <dependency>
       <groupId>commons-io</groupId>
       <artifactId>commons-io</artifactId>

diff --git a/spark-hpcc/pom.xml b/spark-hpcc/pom.xml
@@ -88,6 +88,23 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.hpccsystems</groupId>
+      <artifactId>wsclient</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+      <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.antlr</groupId>
+          <artifactId>antlr4-runtime</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.antlr</groupId>
+          <artifactId>antlr4</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <dependency>
       <groupId>org.hpccsystems</groupId>
       <artifactId>dfsclient</artifactId>
@@ -150,6 +167,14 @@
     </plugins>
   </build>
   <profiles>
+    <profile>
+      <id>spark24</id>
+      <properties>
+        <spark.runtime.version>2.4.6</spark.runtime.version>
+        <scala.binary.version>2.11</scala.binary.version>
+        <scala.version>2.11</scala.version>
+      </properties>
+    </profile>
     <profile>
         <id>spark33</id>
         <properties>

diff --git a/spark-hpcc/src/test/java/org/hpccsystems/spark/BaseIntegrationTest.java b/spark-hpcc/src/test/java/org/hpccsystems/spark/BaseIntegrationTest.java
@@ -24,8 +24,9 @@
 import org.apache.spark.SparkConf;
 import org.apache.spark.SparkContext;
 import org.apache.spark.sql.SparkSession;
+import org.hpccsystems.ws.client.BaseRemoteTest;
 
-class BaseIntegrationTest
+class BaseIntegrationTest extends BaseRemoteTest
 {
     static SparkContext sparkContext = null;
 
@@ -147,21 +148,21 @@ public SparkSession getOrCreateSparkSession(SparkConf conf)
 
     public String getHPCCClusterURL()
     {
-        return System.getProperty("hpccconn", "https://eclwatch.default:8010");
+        return connString;
     }
 
     public String getHPCCClusterUser()
     {
-        return System.getProperty("hpccuser", "");
+        return hpccUser;
     }
 
     public String getHPCCClusterPass()
     {
-        return System.getProperty("hpccpass", "");
+        return hpccPass;
     }
 
     public String getThorCluster()
     {
-        return System.getProperty("thorclustername", "data");
+        return thorClusterFileGroup;
     }
 }
diff --git a/wsclient/pom.xml b/wsclient/pom.xml
@@ -99,6 +99,16 @@
       <artifactId>commons-hpcc</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.antlr</groupId>
+      <artifactId>antlr4</artifactId>
+      <version>${antlr.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.antlr</groupId>
+      <artifactId>antlr4-runtime</artifactId>
+      <version>${antlr.version}</version>
+    </dependency>
   </dependencies>
 
  <profiles>