Skip to content

Commit b7c1711

Browse files
committed
bumping down to kotlin 1.9.23 to make lambda's serializable again
1 parent 1b0b316 commit b7c1711

File tree

2 files changed

+4
-2
lines changed
  • buildSrc/src/main/kotlin
  • kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api

2 files changed

+4
-2
lines changed

buildSrc/src/main/kotlin/Versions.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ object Versions : Dsl<Versions> {
22
const val project = "2.0.0-SNAPSHOT"
33
const val kotlinSparkApiGradlePlugin = "2.0.0-SNAPSHOT"
44
const val groupID = "org.jetbrains.kotlinx.spark"
5-
const val kotlin = "2.0.0-Beta5"
5+
// const val kotlin = "2.0.0-Beta5" todo issues with NonSerializable lambdas
6+
const val kotlin = "1.9.23"
67
const val jvmTarget = "8"
78
const val jupyterJvmTarget = "8"
89
inline val spark get() = System.getProperty("spark") as String

kotlin-spark-api/src/main/kotlin/org/jetbrains/kotlinx/spark/api/Dataset.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import org.apache.spark.api.java.function.FlatMapFunction
3434
import org.apache.spark.api.java.function.ForeachFunction
3535
import org.apache.spark.api.java.function.ForeachPartitionFunction
3636
import org.apache.spark.api.java.function.MapFunction
37+
import org.apache.spark.api.java.function.MapPartitionsFunction
3738
import org.apache.spark.api.java.function.ReduceFunction
3839
import org.apache.spark.rdd.RDD
3940
import org.apache.spark.sql.*
@@ -161,7 +162,7 @@ inline fun <T, reified R> Dataset<T>.groupByKey(noinline func: (T) -> R): KeyVal
161162
* Returns a new Dataset that contains the result of applying [func] to each partition.
162163
*/
163164
inline fun <T, reified R> Dataset<T>.mapPartitions(noinline func: (Iterator<T>) -> Iterator<R>): Dataset<R> =
164-
mapPartitions(func, kotlinEncoderFor<R>())
165+
mapPartitions(MapPartitionsFunction(func), kotlinEncoderFor<R>())
165166

166167
/**
167168
* (Kotlin-specific)

0 commit comments

Comments
 (0)