From 77c54abee8841e30d5da01fe433a5712ebc2ac60 Mon Sep 17 00:00:00 2001 From: Aaron Z Date: Mon, 17 Feb 2025 20:44:03 -0500 Subject: [PATCH] updating spark versions --- README.md | 2 +- build.sbt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e8d7e8c..f07267b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The goal of this repo is not to represent every permutation of a json schema -> ## Running -Built for **Spark 3.3.0 & Scala 2.12** Please make sure your cluster matches these versions +Latest updated build for **Spark 3.5.0 & Scala 2.12** Please make sure your cluster matches these versions. ```scala import com.databricks.industry.solutions.json2spark._ diff --git a/build.sbt b/build.sbt index 738d856..3654133 100644 --- a/build.sbt +++ b/build.sbt @@ -1,11 +1,12 @@ lazy val scala212 = "2.12.8" -lazy val sparkVersion = sys.env.getOrElse("SPARK_VERSION", "3.3.0") +lazy val sparkVersion = sys.env.getOrElse("SPARK_VERSION", "3.5.0") ThisBuild / scalaVersion := sys.env.getOrElse("SCALA_VERSION", scala212) ThisBuild / organization := "com.databricks" ThisBuild / organizationName := "Databricks, Inc." lazy val sparkDependencies = Seq( + "com.databricks" % "databricks-jdbc" % "2.7.1", "org.apache.spark" %% "spark-catalyst" % sparkVersion, "org.apache.spark" %% "spark-core" % sparkVersion, "org.apache.spark" %% "spark-mllib" % sparkVersion, @@ -38,7 +39,7 @@ assembly / assemblyShadeRules := Seq( lazy val core = (project in file(".")) .settings( name := "json2spark-schema", - version := "0.0.3", + version := "0.0.4", scalacOptions += "-target:jvm-1.8", libraryDependencies ++= coreDependencies ++ projectDependencies , assemblyJarName := s"${name.value}-${version.value}_assembly.jar",