generateDomainClassesCheck: analogous to scalafmtCheck: safety net for PRs (#239)

* upgrade

* documentation text updates

* generateDomainClassesCheck: analogous to `scalafmtCheck`

Additional fixes:
* fix FileUtils.deleteRecursively
* generateDomainClasses now deletes it's target directory - for real
this time

* readme

* fix tests: allow to 'not delete' the codegen target dir...

...now that 'deleteRecursively' works :)

* run scalafmt to avoid build warnings...

Author: mpollmeier

Diff for: .github/workflows/pr.yml

@@ -2,9 +2,9 @@ name: pr
 on: pull_request
 jobs:
   pr:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 1
       - uses: olafurpg/setup-scala@v10

Diff for: .github/workflows/release.yml

@@ -6,9 +6,9 @@ on:
 jobs:
   release:
     concurrency: release
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - uses: olafurpg/setup-scala@v10

Diff for: README.md

@@ -2,7 +2,16 @@
 
 # overflowdb-codegen
 
-## Configuration overview: TODO
+## generateDomainClasses
+Freshly generates the domain classes to the configured location. If both the schema as well as the dependency tree (which includes the codegen plugin) are unchanged, it will not run. This is to avoid long build times due to unnecessary regeneration and recompilation for consecutive `sbt compile` runs. 
+
+Note: the regular `sbt clean` does not touch/delete/handle the codegen-generated sources. This is on purpose, since the idea is to have the generated domain classes committed to the repository. 
+
+## generateDomainClassesCheck
+Fails if domain classes are not generated with the latest versions. Analogous to `scalafmtCheck`, i.e. run this on PRs.
+
+## Example repositories / builds
+* TODO
 
 ## Disable temporarily
 You can temporarily disable the codegen in your build by setting the environment variable `ODB_CODEGEN_DISABLE=true`. That's useful e.g. if you made some manual changes to the generated files that would otherwise be overridden by the codegen. 

Diff for: codegen/src/main/scala/overflowdb/codegen/CodeGen.scala

@@ -31,7 +31,11 @@ class CodeGen(schema: Schema) {
     this
   }
 
-  def run(outputDir: java.io.File): Seq[java.io.File] = {
+  def run(outputDir: java.io.File, deleteExistingFiles: Boolean = true): Seq[java.io.File] = {
+    println(s"writing domain classes to $outputDir")
+    if (deleteExistingFiles && outputDir.exists)
+      deleteRecursively(outputDir)
+
     warnForDuplicatePropertyDefinitions()
     val _outputDir = outputDir.toScala
     val results =
@@ -1938,6 +1942,13 @@ class CodeGen(schema: Schema) {
                      |$src
                      |""".stripMargin)
   }
+
+  private def deleteRecursively(file: java.io.File): Unit = {
+    if (file.isDirectory)
+      file.listFiles.foreach(deleteRecursively)
+    if (file.exists)
+      file.delete()
+  }
 }
 
 object CodeGen {

Diff for: codegen/src/main/scala/overflowdb/codegen/Formatter.scala

@@ -13,6 +13,7 @@ object Formatter {
       |""".stripMargin
 
   def run(sourceFiles: Seq[File], scalafmtConfig: Option[File]): Unit = {
+    println(s"invoking scalafmt on ${sourceFiles.size} files")
     val configFile: File = scalafmtConfig.getOrElse(
       Files
         .createTempFile("overflowdb-scalafmt", ".conf")

Diff for: integration-tests/schemas/build.sbt

@@ -14,12 +14,11 @@ generateDomainClasses := Def.taskDyn {
 
   if (outputRoot.exists && lastSchemaMd5 == Some(currentSchemaMd5)) {
     Def.task {
-      // inputs did not change, don't regenerate
+      streams.value.log.info("no need to run codegen, inputs did not change")
       FileUtils.listFilesRecursively(outputRoot)
     }
   } else {
     Def.task {
-      FileUtils.deleteRecursively(outputRoot)
       val invoked = (Compile/runMain).toTask(s" CodegenForAllSchemas").value
       lastSchemaMd5(currentSchemaMd5)
       FileUtils.listFilesRecursively(outputRoot)

Diff for: integration-tests/schemas/src/main/scala/CodegenForAllSchemas.scala

@@ -10,6 +10,8 @@ object CodegenForAllSchemas {
       else "scala-2.13"
 
     val outputDir = new File(s"integration-tests/schemas/target/$scalaVersion/odb-codegen")
+    println(s"running CodegenForAllSchemas; deleting outputDir first: $outputDir")
+    deleteRecursively(outputDir)
 
     Seq(
       new TestSchema01,
@@ -21,7 +23,7 @@ object CodegenForAllSchemas {
       new TestSchema05,
       new TestSchema06,
     ).foreach { schema =>
-      new CodeGen(schema.instance).disableScalafmt.run(outputDir)
+      new CodeGen(schema.instance).run(outputDir, deleteExistingFiles = false)
     }
   }
 
@@ -33,4 +35,11 @@ object CodegenForAllSchemas {
     case u: java.net.URLClassLoader => u.getURLs() ++ classpathUrls(cl.getParent)
     case _ => Array.empty
   }
+
+  private def deleteRecursively(file: File): Unit = {
+    if (file.isDirectory)
+      file.listFiles.foreach(deleteRecursively)
+    if (file.exists && !file.delete())
+      throw new java.io.IOException(s"Unable to delete ${file.getAbsolutePath}")
+  }
 }

Diff for: project/FileUtils.scala

@@ -1,7 +1,7 @@
 import java.io.File
 import java.nio.file.Files
 import java.security.{DigestInputStream, MessageDigest}
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters.*
 
 object FileUtils {
 
@@ -11,24 +11,13 @@ object FileUtils {
     }
   }
 
-  def deleteRecursively(root: File): Unit = {
-    if (root.exists) {
-      Files.walk(root.toPath).iterator.asScala.map(_.toFile).collect {
-        case file if (file.isDirectory) => deleteRecursively(file)
-        case file => file.delete()
-      }
-    }
-  }
-
   def md5(roots: File*): String = {
     val md = MessageDigest.getInstance("MD5")
     roots.foreach { root =>
       Files.walk(root.toPath).filter(!_.toFile.isDirectory).forEach { path =>
         val dis = new DigestInputStream(Files.newInputStream(path), md)
-        // fully consume the inputstream
-        while (dis.available > 0) {
-          dis.read
-        }
+        // fully consume the InputStream
+        while (dis.available > 0) dis.read
         dis.close
       }
     }

Diff for: project/build.properties

@@ -1 +1 @@
-sbt.version=1.9.2
+sbt.version=1.9.3

Diff for: sbt-overflowdb/src/main/scala/overflowdb/codegen/sbt/FileUtils.scala

@@ -13,13 +13,11 @@ object FileUtils {
     }
   }
 
-  def deleteRecursively(root: File): Unit = {
-    if (root.exists) {
-      Files.walk(root.toPath).iterator.asScala.map(_.toFile).collect {
-        case file if (file.isDirectory) => deleteRecursively(file)
-        case file => file.delete()
-      }
-    }
+  def deleteRecursively(file: File): Unit = {
+    if (file.isDirectory)
+      file.listFiles.foreach(deleteRecursively)
+    if (file.exists)
+      file.delete()
   }
 
   def md5(roots: File*): String = {

Diff for: sbt-overflowdb/src/main/scala/overflowdb/codegen/sbt/OdbCodegenSbtPlugin.scala

@@ -11,14 +11,16 @@ import scala.util.Try
 object OdbCodegenSbtPlugin extends AutoPlugin {
 
   object autoImport {
-    val generateDomainClasses = taskKey[File]("generate overflowdb domain classes for the given schema - return value is the output root directory")
+    val generateDomainClasses = taskKey[File]("regenerates domain classes for the given schema; return value is the output root directory")
+    val generateDomainClassesCheck = taskKey[Unit]("Fails if domain classes are not generated with the latest versions. Analogous to `scalafmtCheck`, i.e. run this on PRs.")
     val outputDir = settingKey[File]("target directory for the generated domain classes, e.g. `Projects.domainClasses/scalaSource`")
     val classWithSchema = settingKey[String]("class with schema field, e.g. `org.example.MyDomain$`")
     val fieldName = settingKey[String]("(static) field name for schema within the specified `classWithSchema` with schema field, e.g. `org.example.MyDomain$`")
-    val disableFormatting = settingKey[Boolean]("disable scalafmt formatting")
+    val disableFormatting = settingKey[Boolean]("disable automatic scalafmt invocation")
 
     lazy val baseSettings: Seq[Def.Setting[_]] = Seq(
       generateDomainClasses := generateDomainClassesTask.value,
+      generateDomainClassesCheck := generateDomainClassesCheckTask.value,
       generateDomainClasses/disableFormatting := false,
     )
   }
@@ -32,7 +34,7 @@ object OdbCodegenSbtPlugin extends AutoPlugin {
   // a group of settings that are automatically added to projects.
   override val projectSettings = inConfig(Compile)(autoImport.baseSettings)
 
-  lazy val generateDomainClassesTask =
+  lazy val generateDomainClassesTask = {
     Def.taskDyn {
       val classWithSchemaValue = (generateDomainClasses/classWithSchema).value
       val fieldNameValue = (generateDomainClasses/fieldName).value
@@ -49,7 +51,7 @@ object OdbCodegenSbtPlugin extends AutoPlugin {
       }
 
       val schemaAndDependenciesHashFile = target.value / "overflowdb-schema-and-dependencies.md5"
-      val dependenciesFile = target.value / "dependenciesCP.txt"
+      val dependenciesFile = target.value / "dependenciesCP.txt" // includes codegen version!
       IO.write(dependenciesFile, dependencyClasspath.value.mkString(System.lineSeparator))
       lazy val currentSchemaAndDependenciesHash =
         FileUtils.md5(sourceDirectory.value, baseDirectory.value/"build.sbt", dependenciesFile)
@@ -63,11 +65,10 @@ object OdbCodegenSbtPlugin extends AutoPlugin {
           outputDirValue
         }
       } else if (outputDirValue.exists && lastSchemaAndDependenciesHash == Some(currentSchemaAndDependenciesHash)) {
-        // inputs did not change, don't regenerate
+        streams.value.log.info("inputs did not change -> no need to run codegen")
         Def.task { outputDirValue }
       } else {
         Def.task {
-          FileUtils.deleteRecursively(outputDirValue)
           (Compile/runMain).toTask(
             s" overflowdb.codegen.Main --classWithSchema=$classWithSchemaValue --field=$fieldNameValue --out=$outputDirValue $disableFormattingParamMaybe $scalafmtConfigFileMaybe"
           ).value
@@ -76,6 +77,38 @@ object OdbCodegenSbtPlugin extends AutoPlugin {
         }
       }
     }
+  }
+
+  lazy val generateDomainClassesCheckTask = {
+    Def.taskDyn {
+      streams.value.log.info("generateDomainClassesCheck: running codegen for comparison")
+      val classWithSchemaValue = (generateDomainClasses/classWithSchema).value
+      val fieldNameValue = (generateDomainClasses/fieldName).value
+      val outputDirValue = (generateDomainClasses/outputDir).value
+      val tempOutputDir = target.value / "generate-domain-classes-check"
+
+      val disableFormattingParamMaybe =
+        if ((generateDomainClasses/disableFormatting).value) "--noformat"
+        else ""
+
+      val scalafmtConfigFileMaybe = {
+        val file = (generateDomainClasses/scalafmtConfig).value
+        if (file.exists) s"--scalafmtConfig=$file"
+        else ""
+      }
+
+      Def.task {
+        (Compile/runMain).toTask(
+          s" overflowdb.codegen.Main --classWithSchema=$classWithSchemaValue --field=$fieldNameValue --out=$tempOutputDir $disableFormattingParamMaybe $scalafmtConfigFileMaybe"
+        ).value
+
+        val generatedSourcesIdentical = FileUtils.md5(outputDirValue) == FileUtils.md5(tempOutputDir)
+        if (!generatedSourcesIdentical) {
+          throw new MessageOnlyException("there are differences in the generated and the existing sources, please run `generateDomainClasses` to fix this")
+        }
+      }
+    }
+  }
 
 
 }