read resources recursively (#102)

* read resources recursively * add tests
rolang · Dec 1, 2024 · a871e24 · a871e24
1 parent bf7a147
commit a871e24
Show file tree

Hide file tree

Showing 22 changed files with 125 additions and 19 deletions.
diff --git a/modules/core/jvm/src/main/scala-2/dumbo/ResourceFilePath.scala b/modules/core/jvm/src/main/scala-2/dumbo/ResourceFilePath.scala
@@ -22,6 +22,15 @@ final case class ResourceFilePath(value: String) extends AnyVal {
 }
 
 object ResourceFilePath {
+  @scala.annotation.tailrec
+  private def listRec(dirs: List[File], files: List[File]): List[File] =
+    dirs match {
+      case x :: xs =>
+        val (d, f) = x.listFiles().toList.partition(_.isDirectory())
+        listRec(d ::: xs, f ::: files)
+      case Nil => files
+    }
+
   private[dumbo] def fromResourcesDir[F[_]: Sync](location: String): (String, F[List[ResourceFilePath]]) =
     Try(getClass().getClassLoader().getResources(location).asScala.toList) match {
       case Failure(err)                                           => ("", Sync[F].raiseError(err))
@@ -33,10 +42,9 @@ object ResourceFilePath {
           Sync[F].delay {
             val base = Paths.get(url.toURI())
             val resources =
-              new File(base.toString())
-                .list()
-                .map(fileName => ResourceFilePath(s"/$location/$fileName"))
-                .toList
+              listRec(List(new File(base.toString())), Nil).map(f =>
+                ResourceFilePath(s"/$location/${base.relativize(Paths.get(f.getAbsolutePath()))}")
+              )
             resources
           },
         )

diff --git a/modules/core/jvm/src/main/scala-2/dumbo/internal/DumboPlatform.scala b/modules/core/jvm/src/main/scala-2/dumbo/internal/DumboPlatform.scala
@@ -12,7 +12,11 @@ private[dumbo] trait DumboPlatform {
     val (locationInfo, resources) = ResourceFilePath.fromResourcesDir(location)
 
     new DumboWithResourcesPartiallyApplied[F](
-      ResourceReader.embeddedResources(resources, Some(locationInfo))
+      ResourceReader.embeddedResources(
+        readResources = resources,
+        locationInfo = Some(locationInfo),
+        locationRelative = Some(location),
+      )
     )
   }
 }
diff --git a/modules/core/shared/src/main/scala-3/dumbo/ResourceFilePath.scala b/modules/core/shared/src/main/scala-3/dumbo/ResourceFilePath.scala
@@ -38,9 +38,18 @@ object ResourceFilePath:
 
           Expr(resources)
         else
+          @scala.annotation.tailrec
+          def listRec(dirs: List[File], files: List[File]): List[File] =
+            dirs match
+              case x :: xs =>
+                val (d, f) = x.listFiles().toList.partition(_.isDirectory())
+                listRec(d ::: xs, f ::: files)
+              case Nil => files
+
           val base = Paths.get(head.toURI())
-          val resources =
-            new File(base.toString()).list().map(fileName => s"/$location/$fileName").toList
+          val resources = listRec(List(File(base.toString())), Nil).map(f =>
+            s"/$location/${base.relativize(Paths.get(f.getAbsolutePath()))}"
+          )
           Expr(resources)
       case Nil => report.errorAndAbort(s"resource ${location} was not found")
       case multiple =>

diff --git a/modules/core/shared/src/main/scala-3/dumbo/internal/DumboPlatform.scala b/modules/core/shared/src/main/scala-3/dumbo/internal/DumboPlatform.scala
@@ -10,6 +10,12 @@ import dumbo.{DumboWithResourcesPartiallyApplied, ResourceFilePath}
 private[dumbo] trait DumboPlatform {
   inline def withResourcesIn[F[_]: Sync](location: String): DumboWithResourcesPartiallyApplied[F] = {
     val resources = ResourceFilePath.fromResourcesDir(location)
-    new DumboWithResourcesPartiallyApplied[F](ResourceReader.embeddedResources(Sync[F].pure(resources)))
+    new DumboWithResourcesPartiallyApplied[F](
+      ResourceReader.embeddedResources(
+        readResources = Sync[F].pure(resources),
+        locationInfo = Some(location),
+        locationRelative = Some(location),
+      )
+    )
   }
 }
diff --git a/modules/core/shared/src/main/scala/dumbo/Dumbo.scala b/modules/core/shared/src/main/scala/dumbo/Dumbo.scala
@@ -222,13 +222,19 @@ class Dumbo[F[_]: Sync: Console](
       version = source.versionText,
       description = source.scriptDescription,
       `type` = "SQL",
-      script = source.path.fileName,
+      script = historyScriptPath(source),
       checksum = Some(source.checksum),
       executionTimeMs = duration.toMillis.toInt,
       success = true,
     )
   }
 
+  private def historyScriptPath(resource: ResourceFile) =
+    resReader.locationRel match {
+      case Some(loc) => resource.path.value.stripPrefix(s"/$loc/")
+      case _         => resource.fileName
+    }
+
   private def validationGuard(session: Session[F], resources: ResourceFiles) =
     if (resources.nonEmpty) {
       session
@@ -446,13 +452,13 @@ class Dumbo[F[_]: Sync: Console](
     resources: ResourceFiles,
   ): ValidatedNec[DumboValidationException, Unit] = {
     val versionedMap: Map[String, ResourceFile] = resources.versioned.map { case (v, f) => (v.text, f) }.toMap
-    val repeatablesFileNames: Set[String]       = resources.repeatable.map(_._2.fileName).toSet
+    val repeatablesScriptNames: Set[String]     = resources.repeatable.map(_._2.path.value).toSet
 
     history
       .filter(_.`type` == "SQL")
       .traverse { h =>
         versionedMap.get(h.version.getOrElse("")) match {
-          case None if !repeatablesFileNames.contains(h.script) =>
+          case None if !repeatablesScriptNames.exists(_.endsWith(h.script)) =>
             new DumboValidationException(s"Detected applied migration not resolved locally ${h.script}")
               .invalidNec[Unit]
 

diff --git a/modules/core/shared/src/main/scala/dumbo/internal/ResourcesReader.scala b/modules/core/shared/src/main/scala/dumbo/internal/ResourcesReader.scala
@@ -4,13 +4,19 @@
 
 package dumbo.internal
 
+import java.io.File
+
 import cats.effect.Sync
 import cats.implicits.*
 import dumbo.ResourceFilePath
 import fs2.io.file.{Files as Fs2Files, Flags, Path}
 import fs2.{Stream, text}
 
 private[dumbo] trait ResourceReader[F[_]] {
+  // relative location info e.g. "db/migration"
+  def locationRel: Option[String]
+
+  // relative or absolute location info e.g. "file:/project/resources/db/migration"
   def location: Option[String]
 
   def list: fs2.Stream[F, ResourceFilePath]
@@ -28,12 +34,22 @@ private[dumbo] object ResourceReader {
 
     @inline def absolutePath(p: Path) = if (p.isAbsolute) p else base / p
 
+    @scala.annotation.tailrec
+    def listRec(dirs: List[File], files: List[File]): List[File] =
+      dirs match {
+        case x :: xs =>
+          val (d, f) = x.listFiles().toList.partition(_.isDirectory())
+          listRec(d ::: xs, f ::: files)
+        case Nil => files
+      }
+
     new ResourceReader[F] {
-      override val location: Option[String] = Some(absolutePath(sourceDir).toString)
+      override val locationRel: Option[String] = Some(sourceDir.toString)
+      override val location: Option[String]    = Some(absolutePath(sourceDir).toString)
       override def list: Stream[F, ResourceFilePath] =
-        Fs2Files[F]
-          .list(absolutePath(sourceDir))
-          .map(p => ResourceFilePath(p.toString))
+        Stream.emits(
+          listRec(List(new File(absolutePath(sourceDir).toString)), Nil).map(f => ResourceFilePath(f.getPath()))
+        )
 
       override def readUtf8Lines(path: ResourceFilePath): Stream[F, String] =
         Fs2Files[F].readUtf8Lines(absolutePath(Path(path.value)))
@@ -48,8 +64,11 @@ private[dumbo] object ResourceReader {
   def embeddedResources[F[_]: Sync](
     readResources: F[List[ResourceFilePath]],
     locationInfo: Option[String] = None,
+    locationRelative: Option[String] = None,
   ): ResourceReader[F] =
     new ResourceReader[F] {
+      override val locationRel: Option[String] = locationRelative
+
       override val location: Option[String] = locationInfo
 
       override def list: Stream[F, ResourceFilePath] = Stream.evals(readResources)

diff --git a/modules/sample-lib/src/main/resources/sample_lib/sub_dir/V3__test_c.sql b/modules/sample-lib/src/main/resources/sample_lib/sub_dir/V3__test_c.sql
@@ -0,0 +1 @@
+CREATE TABLE test_c();
diff --git a/modules/sample-lib/src/main/resources/sample_lib/sub_dir/sub_sub_dir/V4__test_d.sql b/modules/sample-lib/src/main/resources/sample_lib/sub_dir/sub_sub_dir/V4__test_d.sql
@@ -0,0 +1 @@
+CREATE TABLE test_d();
diff --git a/modules/test-lib/src/main/scala/TestLib.scala b/modules/test-lib/src/main/scala/TestLib.scala
@@ -1,8 +1,10 @@
 import dumbo.ResourceFilePath
 
 @main def run(args: String*) =
-  val resources = ResourceFilePath.fromResourcesDir("sample_lib")
-  val expected = List(
+  val resources = ResourceFilePath.fromResourcesDir("sample_lib").map(_.value).toSet
+  val expected = Set(
+    "/sample_lib/sub_dir/sub_sub_dir/V4__test_d.sql",
+    "/sample_lib/sub_dir/V3__test_c.sql",
     "/sample_lib/V1__test.sql",
     "/sample_lib/V2__test_b.sql",
   )

diff --git a/modules/tests-flyway/src/test/scala/DumboFlywaySpec.scala b/modules/tests-flyway/src/test/scala/DumboFlywaySpec.scala
@@ -138,6 +138,19 @@ trait DumboFlywaySpec extends ffstest.FTest {
     } yield ()
   }
 
+  dbTest("Compatible with nested directories") {
+    val schema = "schema_1"
+
+    for {
+      _             <- flywayMigrate(schema, Path("db/nested")).map(r => assert(r.migrationsExecuted == 6))
+      historyFlyway <- loadHistory(schema)
+      _             <- dropSchemas
+      _             <- dumboMigrate(schema, dumboWithResources("db/nested")).map(r => assert(r.migrationsExecuted == 6))
+      historyDumbo  <- loadHistory(schema)
+      _              = assertEqualHistory(historyDumbo, historyFlyway)
+    } yield ()
+  }
+
   dbTest("Dumbo updates history entry of latest unsucessfully applied migration by Flyway") {
     // run on CockroachDb only just because it was the easiest way to reproduce a history record for an unsuccessfully applied migration with Flyway
     if (db == Db.CockroachDb) {

diff --git a/modules/tests/shared/src/test/non_resource/db/test_1/V2__non_resource.sql b/modules/tests/shared/src/test/non_resource/db/test_1/V2__non_resource.sql
@@ -0,0 +1 @@
+CREATE TABLE test (id SERIAL PRIMARY KEY);
diff --git a/modules/tests/shared/src/test/non_resource/db/test_1/sub/R__b.sql b/modules/tests/shared/src/test/non_resource/db/test_1/sub/R__b.sql
diff --git a/modules/tests/shared/src/test/non_resource/db/test_1/sub/V4__non_resource.sql b/modules/tests/shared/src/test/non_resource/db/test_1/sub/V4__non_resource.sql
diff --git a/modules/tests/shared/src/test/non_resource/db/test_1/sub/sub_sub/R__a.sql b/modules/tests/shared/src/test/non_resource/db/test_1/sub/sub_sub/R__a.sql
diff --git a/modules/tests/shared/src/test/non_resource/db/test_1/sub/sub_sub/V3__non_resource.sql b/modules/tests/shared/src/test/non_resource/db/test_1/sub/sub_sub/V3__non_resource.sql
diff --git a/modules/tests/shared/src/test/resources/db/nested/V1__test.sql b/modules/tests/shared/src/test/resources/db/nested/V1__test.sql
@@ -0,0 +1 @@
+CREATE TABLE test_a();
diff --git a/modules/tests/shared/src/test/resources/db/nested/V2__test.sql b/modules/tests/shared/src/test/resources/db/nested/V2__test.sql
@@ -0,0 +1 @@
+CREATE TABLE test_b();
diff --git a/modules/tests/shared/src/test/resources/db/nested/sub/R__b.sql b/modules/tests/shared/src/test/resources/db/nested/sub/R__b.sql
@@ -0,0 +1 @@
+CREATE TABLE test_f();
diff --git a/modules/tests/shared/src/test/resources/db/nested/sub/V4__test.sql b/modules/tests/shared/src/test/resources/db/nested/sub/V4__test.sql
@@ -0,0 +1 @@
+CREATE TABLE test_d();
diff --git a/modules/tests/shared/src/test/resources/db/nested/sub/sub_sub/R__a.sql b/modules/tests/shared/src/test/resources/db/nested/sub/sub_sub/R__a.sql
@@ -0,0 +1 @@
+CREATE TABLE test_e();
diff --git a/modules/tests/shared/src/test/resources/db/nested/sub/sub_sub/V3__test.sql b/modules/tests/shared/src/test/resources/db/nested/sub/sub_sub/V3__test.sql
@@ -0,0 +1 @@
+CREATE TABLE test_c();
diff --git a/modules/tests/shared/src/test/scala/DumboResourcesSpec.scala b/modules/tests/shared/src/test/scala/DumboResourcesSpec.scala
@@ -29,14 +29,39 @@ class DumboResourcesSpec extends ffstest.FTest {
     } yield ()
   }
 
+  test("list migration files from resources with subirectories") {
+    for {
+      files <- dumboWithResources("db/nested").listMigrationFiles
+      _ = files match {
+            case Valid(files) =>
+              assert(
+                files.sorted.map(f => (f.version, f.path.fileName.toString)) == List(
+                  (ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__test.sql"),
+                  (ResourceVersion.Versioned("2", NonEmptyList.of(2)), "V2__test.sql"),
+                  (ResourceVersion.Versioned("3", NonEmptyList.of(3)), "V3__test.sql"),
+                  (ResourceVersion.Versioned("4", NonEmptyList.of(4)), "V4__test.sql"),
+                  (ResourceVersion.Repeatable("a"), "R__a.sql"),
+                  (ResourceVersion.Repeatable("b"), "R__b.sql"),
+                )
+              )
+            case Invalid(errs) => fail(errs.toList.mkString("\n"))
+          }
+    } yield ()
+  }
+
   test("list migration files from relative path") {
     for {
       files <- Dumbo.withFilesIn[IO](Path("modules/tests/shared/src/test/non_resource/db/test_1")).listMigrationFiles
       _ = files match {
             case Valid(files) =>
               assert(
                 files.sorted.map(f => (f.version, f.path.fileName.toString)) == List(
-                  (ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__non_resource.sql")
+                  (ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__non_resource.sql"),
+                  (ResourceVersion.Versioned("2", NonEmptyList.of(2)), "V2__non_resource.sql"),
+                  (ResourceVersion.Versioned("3", NonEmptyList.of(3)), "V3__non_resource.sql"),
+                  (ResourceVersion.Versioned("4", NonEmptyList.of(4)), "V4__non_resource.sql"),
+                  (ResourceVersion.Repeatable("a"), "R__a.sql"),
+                  (ResourceVersion.Repeatable("b"), "R__b.sql"),
                 )
               )
             case Invalid(errs) => fail(errs.toList.mkString("\n"))
@@ -52,7 +77,12 @@ class DumboResourcesSpec extends ffstest.FTest {
             case Valid(files) =>
               assert(
                 files.sorted.map(f => (f.version, f.path.fileName.toString)) == List(
-                  (ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__non_resource.sql")
+                  (ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__non_resource.sql"),
+                  (ResourceVersion.Versioned("2", NonEmptyList.of(2)), "V2__non_resource.sql"),
+                  (ResourceVersion.Versioned("3", NonEmptyList.of(3)), "V3__non_resource.sql"),
+                  (ResourceVersion.Versioned("4", NonEmptyList.of(4)), "V4__non_resource.sql"),
+                  (ResourceVersion.Repeatable("a"), "R__a.sql"),
+                  (ResourceVersion.Repeatable("b"), "R__b.sql"),
                 )
               )
             case Invalid(errs) => fail(errs.toList.mkString("\n"))