now working with spark-connect and scala 2.13

Jolanrensen · Jolanrensen · commit 71f115a9fa0e · 2024-06-18T12:45:13.000+02:00
diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt
@@ -1,9 +1,12 @@
+import org.gradle.jvm.toolchain.JavaLanguageVersion
+
 object Versions : Dsl<Versions> {
     const val project = "2.0.0-SNAPSHOT"
     const val kotlinSparkApiGradlePlugin = "2.0.0-SNAPSHOT"
     const val groupID = "org.jetbrains.kotlinx.spark"
     const val kotlin = "2.0.0-RC3"
-    const val jvmTarget = "8"
+    const val jvmTarget = "1.8"
+    val jvmLanguageVersion = JavaLanguageVersion.of(8)
     const val jupyterJvmTarget = "8"
     inline val spark get() = System.getProperty("spark") as String
     inline val scala get() = System.getProperty("scala") as String
diff --git a/compiler-plugin/build.gradle.kts b/compiler-plugin/build.gradle.kts
@@ -1,3 +1,4 @@
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
@@ -74,13 +75,17 @@ tasks.withType<KotlinCompile>().configureEach {
 
 kotlin {
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(8)
+        languageVersion = Versions.jvmLanguageVersion
+    }
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
 java {
     toolchain {
         languageVersion = JavaLanguageVersion.of(8)
     }
+    targetCompatibility = JavaVersion.toVersion(Versions.jvmTarget)
 }
 
 val generateTests by tasks.creating(JavaExec::class) {
diff --git a/examples/build.gradle.kts b/examples/build.gradle.kts
@@ -1,4 +1,4 @@
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 plugins {
     // Needs to be installed in the local maven repository or have the bootstrap jar on the classpath
@@ -49,8 +49,10 @@ dependencies {
 }
 
 kotlin {
-    jvmToolchain(8)
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
+        languageVersion = Versions.jvmLanguageVersion
+    }
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
diff --git a/gradle-plugin/build.gradle.kts b/gradle-plugin/build.gradle.kts
@@ -1,5 +1,8 @@
 @file:Suppress("UnstableApiUsage")
 
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
+
+
 plugins {
     `java-gradle-plugin`
     kotlin
@@ -51,7 +54,10 @@ dependencies {
 
 kotlin {
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
+        languageVersion = Versions.jvmLanguageVersion
+    }
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
 
diff --git a/gradle/bootstraps/compiler-plugin.jar b/gradle/bootstraps/compiler-plugin.jar
diff --git a/gradle/bootstraps/gradle-plugin.jar b/gradle/bootstraps/gradle-plugin.jar
diff --git a/kotlin-spark-api/build.gradle.kts b/kotlin-spark-api/build.gradle.kts
@@ -4,6 +4,7 @@ import com.igormaznitsa.jcp.gradle.JcpTask
 import com.vanniktech.maven.publish.JavadocJar.Dokka
 import com.vanniktech.maven.publish.KotlinJvm
 import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 plugins {
     kotlin
@@ -66,6 +67,9 @@ dependencies {
             kafkaStreamsTestUtils,
             sparkMl,
         )
+
+        compileOnly(scalaLibrary)
+        testCompileOnly(scalaLibrary)
     }
 }
 
@@ -154,8 +158,11 @@ tasks.compileTestKotlin {
 }
 
 kotlin {
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
+    }
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
+        languageVersion = Versions.jvmLanguageVersion
     }
 }
 
diff --git a/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt b/kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Encoding.kt
@@ -76,6 +76,7 @@ import kotlin.reflect.jvm.javaGetter
 import kotlin.reflect.jvm.javaMethod
 import kotlin.reflect.jvm.jvmName
 import kotlin.reflect.typeOf
+import scala.collection.immutable.Seq as ImmutableSeq
 
 fun <T : Any> kotlinEncoderFor(
     kClass: KClass<T>,
@@ -683,7 +684,7 @@ object KotlinTypeInference : Serializable {
 
                 JavaBeanEncoder<Any>(
                     ClassTag.apply(jClass),
-                    fields.asScalaSeq(),
+                    fields.asScalaSeq() as ImmutableSeq<EncoderField>?,
                 )
             }
         }
diff --git a/scala-helpers/build.gradle.kts b/scala-helpers/build.gradle.kts
@@ -21,8 +21,7 @@ repositories {
 dependencies {
 
     Dependencies {
-        api(
-            scalaLibrary,
+        implementation(
             reflect,
         )
 
@@ -32,18 +31,20 @@ dependencies {
         implementation(
 //            sparkSql, not needed atm
         )
+        compileOnly(scalaLibrary)
+        testCompileOnly(scalaLibrary)
     }
 }
 
 java {
     toolchain {
         if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8
-            languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
-
+            languageVersion = Versions.jvmLanguageVersion
         } else if (Versions.jvmTarget == "1.8" || Versions.jvmTarget == "8") {
             languageVersion = JavaLanguageVersion.of(8)
         }
     }
+    targetCompatibility = JavaVersion.toVersion(Versions.jvmTarget)
 }
 
 tasks.withType<ScalaCompile> {
@@ -54,9 +55,12 @@ tasks.withType<ScalaCompile> {
     }
 }
 
-val scalaMainSources = sourceSets.main.get().scala.sourceDirectories
+val scalaMainSources =
+    sourceSets.main
+        .get()
+        .scala.sourceDirectories
 
-val preprocessMain by tasks.creating(JcpTask::class)  {
+val preprocessMain by tasks.creating(JcpTask::class) {
     sources = scalaMainSources
     clearTarget = true
     fileExtensions = listOf("scala")
@@ -99,4 +103,4 @@ mavenPublishing {
 val skipScalaOnlyDependent = System.getProperty("skipScalaOnlyDependent").toBoolean()
 tasks
     .filter { "publish" in it.name }
-    .forEach { it.onlyIf { !skipScalaOnlyDependent } }
+    .forEach { it.onlyIf { !skipScalaOnlyDependent } }
diff --git a/scala-tuples-in-kotlin/build.gradle.kts b/scala-tuples-in-kotlin/build.gradle.kts
@@ -3,9 +3,7 @@
 import com.vanniktech.maven.publish.JavadocJar.Dokka
 import com.vanniktech.maven.publish.KotlinJvm
 import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask
-import org.jetbrains.dokka.gradle.DokkaTask
-import org.jetbrains.dokka.gradle.DokkaTaskPartial
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 plugins {
     scala
@@ -30,24 +28,26 @@ dependencies {
     Dependencies {
         implementation(
             kotlinStdLib,
-            scalaLibrary,
         )
         testImplementation(
             kotest,
             atrium,
             kotlinTest,
         )
+        compileOnly(scalaLibrary)
+        testCompileOnly(scalaLibrary)
     }
 }
 
-
 kotlin {
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)
+        languageVersion = Versions.jvmLanguageVersion
+    }
+    compilerOptions {
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
 
-
 tasks.withType<AbstractDokkaLeafTask> {
     dokkaSourceSets {
         all {
diff --git a/spark-connect-examples/build.gradle.kts b/spark-connect-examples/build.gradle.kts
@@ -5,6 +5,7 @@ plugins {
     id("org.jetbrains.kotlinx.spark.api")
     kotlin("jvm")
     application
+    shadow
 }
 
 // run with `./gradlew run`
@@ -44,21 +45,21 @@ dependencies {
         implementation(sparkConnectClient)
 
         implementation(kotlinDateTime)
+
+        compileOnly(scalaLibrary)
     }
 }
 
-// spark-connect seems to work well with java 17 as client and java 1.8 as server
-// also set gradle and your project sdk to java 17
 kotlin {
     jvmToolchain {
-        languageVersion = JavaLanguageVersion.of(17)
+        languageVersion = JavaLanguageVersion.of(17)//Versions.jvmLanguageVersion
     }
     compilerOptions {
-        jvmTarget = JvmTarget.JVM_17
+        jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)
     }
 }
 
 tasks.withType<JavaCompile> {
-    sourceCompatibility = JavaVersion.VERSION_17.toString()
-    targetCompatibility = JavaVersion.VERSION_17.toString()
+    sourceCompatibility = JavaVersion.VERSION_1_8.toString()
+    targetCompatibility = JavaVersion.VERSION_1_8.toString()
 }
diff --git a/spark-connect-examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/Main.kt b/spark-connect-examples/src/main/kotlin/org/jetbrains/kotlinx/spark/examples/Main.kt
@@ -5,30 +5,39 @@ import org.jetbrains.kotlinx.spark.api.plugin.annotations.Sparkify
 import org.jetbrains.kotlinx.spark.api.showDS
 import org.jetbrains.kotlinx.spark.api.toList
 import org.jetbrains.kotlinx.spark.api.tuples.X
-import org.jetbrains.kotlinx.spark.api.tuples.t
 import org.jetbrains.kotlinx.spark.api.withSparkConnect
 import scala.Tuple2
 import java.time.LocalDate
 
-// run with `./gradlew run` or set VM options: "--add-opens=java.base/java.nio=ALL-UNNAMED" in the IDE
+// run with `./gradlew runShadow` or set VM options: "--add-opens=java.base/java.nio=ALL-UNNAMED" in the IDE
 fun main() =
     withSparkConnect("sc://localhost") {
         val classFinder = REPLClassDirMonitor("/mnt/data/Projects/kotlin-spark-api/spark-connect-examples/build/classes")
         spark.registerClassFinder(classFinder)
-        spark.addArtifact("/mnt/data/Projects/kotlin-spark-api/spark-connect-examples/build/libs/spark-connect-examples-2.0.0-SNAPSHOT.jar")
+        // make jar first, preferably a fat jar with shadow, but be careful it doesn't contain scala depencencies
+        spark.addArtifact("/mnt/data/Projects/kotlin-spark-api/spark-connect-examples/build/libs/spark-connect-examples-2.0.0-SNAPSHOT-all.jar")
+
 
         val data =
             listOf(
                 Person("Alice", 25, LocalDate.of(1996, 1, 1), "Alice" X Address("1 Main St", "Springfield", "IL", 62701)),
                 Person("Bob", 30, LocalDate.of(1991, 1, 1), "Bob" X Address("2 Main St", "Springfield", "IL", 62701)),
-                Person("Charlie", 35, LocalDate.of(1986, 1, 1), "Charlie" X Address("3 Main St", "Springfield", "IL", 62701)),
+                Person(
+                    "Charlie",
+                    35,
+                    LocalDate.of(1986, 1, 1),
+                    "Charlie" X Address("3 Main St", "Springfield", "IL", 62701),
+                ),
             )
 
         val ds = data.toDS().showDS()
 
-        ds.toList<Person>().forEach {
-            println(it)
-        }
+        ds
+            .filter { it.age > 26 }
+            .toList<Person>()
+            .forEach {
+                println(it)
+            }
     }
 
 @Sparkify

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+import org.jetbrains.kotlin.gradle.dsl.JvmTarget`
`1`	`2`	`import org.jetbrains.kotlin.gradle.dsl.KotlinVersion`
`2`	`3`	`import org.jetbrains.kotlin.gradle.tasks.KotlinCompile`
`3`	`4`
`@@ -74,13 +75,17 @@ tasks.withType<KotlinCompile>().configureEach {`
`74`	`75`
`75`	`76`	`kotlin {`
`76`	`77`	`jvmToolchain {`
`77`		`- languageVersion = JavaLanguageVersion.of(8)`
	`78`	`+ languageVersion = Versions.jvmLanguageVersion`
	`79`	`+ }`
	`80`	`+ compilerOptions {`
	`81`	`+ jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)`
`78`	`82`	`}`
`79`	`83`	`}`
`80`	`84`	`java {`
`81`	`85`	`toolchain {`
`82`	`86`	`languageVersion = JavaLanguageVersion.of(8)`
`83`	`87`	`}`
	`88`	`+ targetCompatibility = JavaVersion.toVersion(Versions.jvmTarget)`
`84`	`89`	`}`
`85`	`90`
`86`	`91`	`val generateTests by tasks.creating(JavaExec::class) {`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile`
	`1`	`+import org.jetbrains.kotlin.gradle.dsl.JvmTarget`
`2`	`2`
`3`	`3`	`plugins {`
`4`	`4`	`// Needs to be installed in the local maven repository or have the bootstrap jar on the classpath`
`@@ -49,8 +49,10 @@ dependencies {`
`49`	`49`	`}`
`50`	`50`
`51`	`51`	`kotlin {`
`52`		`- jvmToolchain(8)`
`53`	`52`	`jvmToolchain {`
`54`		`- languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)`
	`53`	`+ languageVersion = Versions.jvmLanguageVersion`
	`54`	`+ }`
	`55`	`+ compilerOptions {`
	`56`	`+ jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)`
`55`	`57`	`}`
`56`	`58`	`}`
Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,7 @@ import com.igormaznitsa.jcp.gradle.JcpTask`
`4`	`4`	`import com.vanniktech.maven.publish.JavadocJar.Dokka`
`5`	`5`	`import com.vanniktech.maven.publish.KotlinJvm`
`6`	`6`	`import org.jetbrains.dokka.gradle.AbstractDokkaLeafTask`
	`7`	`+import org.jetbrains.kotlin.gradle.dsl.JvmTarget`
`7`	`8`
`8`	`9`	`plugins {`
`9`	`10`	`kotlin`
`@@ -66,6 +67,9 @@ dependencies {`
`66`	`67`	`kafkaStreamsTestUtils,`
`67`	`68`	`sparkMl,`
`68`	`69`	`)`
	`70`	`+`
	`71`	`+ compileOnly(scalaLibrary)`
	`72`	`+ testCompileOnly(scalaLibrary)`
`69`	`73`	`}`
`70`	`74`	`}`
`71`	`75`
`@@ -154,8 +158,11 @@ tasks.compileTestKotlin {`
`154`	`158`	`}`
`155`	`159`
`156`	`160`	`kotlin {`
	`161`	`+ compilerOptions {`
	`162`	`+ jvmTarget = JvmTarget.fromTarget(Versions.jvmTarget)`
	`163`	`+ }`
`157`	`164`	`jvmToolchain {`
`158`		`- languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)`
	`165`	`+ languageVersion = Versions.jvmLanguageVersion`
`159`	`166`	`}`
`160`	`167`	`}`
`161`	`168`
Original file line number	Diff line number	Diff line change
`@@ -76,6 +76,7 @@ import kotlin.reflect.jvm.javaGetter`
`76`	`76`	`import kotlin.reflect.jvm.javaMethod`
`77`	`77`	`import kotlin.reflect.jvm.jvmName`
`78`	`78`	`import kotlin.reflect.typeOf`
	`79`	`+import scala.collection.immutable.Seq as ImmutableSeq`
`79`	`80`
`80`	`81`	`fun <T : Any> kotlinEncoderFor(`
`81`	`82`	`kClass: KClass<T>,`
`@@ -683,7 +684,7 @@ object KotlinTypeInference : Serializable {`
`683`	`684`
`684`	`685`	`JavaBeanEncoder<Any>(`
`685`	`686`	`ClassTag.apply(jClass),`
`686`		`- fields.asScalaSeq(),`
	`687`	`+ fields.asScalaSeq() as ImmutableSeq<EncoderField>?,`
`687`	`688`	`)`
`688`	`689`	`}`
`689`	`690`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,8 +21,7 @@ repositories {`
`21`	`21`	`dependencies {`
`22`	`22`
`23`	`23`	`Dependencies {`
`24`		`- api(`
`25`		`- scalaLibrary,`
	`24`	`+ implementation(`
`26`	`25`	`reflect,`
`27`	`26`	`)`
`28`	`27`
`@@ -32,18 +31,20 @@ dependencies {`
`32`	`31`	`implementation(`
`33`	`32`	`// sparkSql, not needed atm`
`34`	`33`	`)`
	`34`	`+ compileOnly(scalaLibrary)`
	`35`	`+ testCompileOnly(scalaLibrary)`
`35`	`36`	`}`
`36`	`37`	`}`
`37`	`38`
`38`	`39`	`java {`
`39`	`40`	`toolchain {`
`40`	`41`	`if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8`
`41`		`- languageVersion = JavaLanguageVersion.of(Versions.jvmTarget)`
`42`		`-`
	`42`	`+ languageVersion = Versions.jvmLanguageVersion`
`43`	`43`	`} else if (Versions.jvmTarget == "1.8" \|\| Versions.jvmTarget == "8") {`
`44`	`44`	`languageVersion = JavaLanguageVersion.of(8)`
`45`	`45`	`}`
`46`	`46`	`}`
	`47`	`+ targetCompatibility = JavaVersion.toVersion(Versions.jvmTarget)`
`47`	`48`	`}`
`48`	`49`
`49`	`50`	`tasks.withType<ScalaCompile> {`
`@@ -54,9 +55,12 @@ tasks.withType<ScalaCompile> {`
`54`	`55`	`}`
`55`	`56`	`}`
`56`	`57`
`57`		`-val scalaMainSources = sourceSets.main.get().scala.sourceDirectories`
	`58`	`+val scalaMainSources =`
	`59`	`+ sourceSets.main`
	`60`	`+ .get()`
	`61`	`+ .scala.sourceDirectories`
`58`	`62`
`59`		`-val preprocessMain by tasks.creating(JcpTask::class) {`
	`63`	`+val preprocessMain by tasks.creating(JcpTask::class) {`
`60`	`64`	`sources = scalaMainSources`
`61`	`65`	`clearTarget = true`
`62`	`66`	`fileExtensions = listOf("scala")`
`@@ -99,4 +103,4 @@ mavenPublishing {`
`99`	`103`	`val skipScalaOnlyDependent = System.getProperty("skipScalaOnlyDependent").toBoolean()`
`100`	`104`	`tasks`
`101`	`105`	`.filter { "publish" in it.name }`
`102`		`- .forEach { it.onlyIf { !skipScalaOnlyDependent } }`
	`106`	`+ .forEach { it.onlyIf { !skipScalaOnlyDependent } }`