bumping down to kotlin 1.9.23 to make lambda's serializable again

Author: Jolanrensen

buildSrc/src/main/kotlin/Versions.kt

@@ -2,7 +2,8 @@ object Versions : Dsl<Versions> {
     const val project = "2.0.0-SNAPSHOT"
     const val kotlinSparkApiGradlePlugin = "2.0.0-SNAPSHOT"
     const val groupID = "org.jetbrains.kotlinx.spark"
-    const val kotlin = "2.0.0-Beta5"
+//    const val kotlin = "2.0.0-Beta5" todo issues with NonSerializable lambdas
+    const val kotlin = "1.9.23"
     const val jvmTarget = "8"
     const val jupyterJvmTarget = "8"
     inline val spark get() = System.getProperty("spark") as String

kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt

@@ -34,6 +34,7 @@ import org.apache.spark.api.java.function.FlatMapFunction
 import org.apache.spark.api.java.function.ForeachFunction
 import org.apache.spark.api.java.function.ForeachPartitionFunction
 import org.apache.spark.api.java.function.MapFunction
+import org.apache.spark.api.java.function.MapPartitionsFunction
 import org.apache.spark.api.java.function.ReduceFunction
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.*
@@ -161,7 +162,7 @@ inline fun <T, reified R> Dataset<T>.groupByKey(noinline func: (T) -> R): KeyVal
  * Returns a new Dataset that contains the result of applying [func] to each partition.
  */
 inline fun <T, reified R> Dataset<T>.mapPartitions(noinline func: (Iterator<T>) -> Iterator<R>): Dataset<R> =
-    mapPartitions(func, kotlinEncoderFor<R>())
+    mapPartitions(MapPartitionsFunction(func), kotlinEncoderFor<R>())
 
 /**
  * (Kotlin-specific)