Skip to content

Commit a871e24

Browse files
rolangrolang
authored
read resources recursively (#102)
* read resources recursively * add tests
1 parent bf7a147 commit a871e24

File tree

22 files changed

+125
-19
lines changed

22 files changed

+125
-19
lines changed

‎modules/core/jvm/src/main/scala-2/dumbo/ResourceFilePath.scala

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,15 @@ final case class ResourceFilePath(value: String) extends AnyVal {
2222
}
2323

2424
object ResourceFilePath {
25+
@scala.annotation.tailrec
26+
private def listRec(dirs: List[File], files: List[File]): List[File] =
27+
dirs match {
28+
case x :: xs =>
29+
val (d, f) = x.listFiles().toList.partition(_.isDirectory())
30+
listRec(d ::: xs, f ::: files)
31+
case Nil => files
32+
}
33+
2534
private[dumbo] def fromResourcesDir[F[_]: Sync](location: String): (String, F[List[ResourceFilePath]]) =
2635
Try(getClass().getClassLoader().getResources(location).asScala.toList) match {
2736
case Failure(err) => ("", Sync[F].raiseError(err))
@@ -33,10 +42,9 @@ object ResourceFilePath {
3342
Sync[F].delay {
3443
val base = Paths.get(url.toURI())
3544
val resources =
36-
new File(base.toString())
37-
.list()
38-
.map(fileName => ResourceFilePath(s"/$location/$fileName"))
39-
.toList
45+
listRec(List(new File(base.toString())), Nil).map(f =>
46+
ResourceFilePath(s"/$location/${base.relativize(Paths.get(f.getAbsolutePath()))}")
47+
)
4048
resources
4149
},
4250
)

‎modules/core/jvm/src/main/scala-2/dumbo/internal/DumboPlatform.scala

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ private[dumbo] trait DumboPlatform {
1212
val (locationInfo, resources) = ResourceFilePath.fromResourcesDir(location)
1313

1414
new DumboWithResourcesPartiallyApplied[F](
15-
ResourceReader.embeddedResources(resources, Some(locationInfo))
15+
ResourceReader.embeddedResources(
16+
readResources = resources,
17+
locationInfo = Some(locationInfo),
18+
locationRelative = Some(location),
19+
)
1620
)
1721
}
1822
}

‎modules/core/shared/src/main/scala-3/dumbo/ResourceFilePath.scala

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,18 @@ object ResourceFilePath:
3838

3939
Expr(resources)
4040
else
41+
@scala.annotation.tailrec
42+
def listRec(dirs: List[File], files: List[File]): List[File] =
43+
dirs match
44+
case x :: xs =>
45+
val (d, f) = x.listFiles().toList.partition(_.isDirectory())
46+
listRec(d ::: xs, f ::: files)
47+
case Nil => files
48+
4149
val base = Paths.get(head.toURI())
42-
val resources =
43-
new File(base.toString()).list().map(fileName => s"/$location/$fileName").toList
50+
val resources = listRec(List(File(base.toString())), Nil).map(f =>
51+
s"/$location/${base.relativize(Paths.get(f.getAbsolutePath()))}"
52+
)
4453
Expr(resources)
4554
case Nil => report.errorAndAbort(s"resource ${location} was not found")
4655
case multiple =>

‎modules/core/shared/src/main/scala-3/dumbo/internal/DumboPlatform.scala

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ import dumbo.{DumboWithResourcesPartiallyApplied, ResourceFilePath}
1010
private[dumbo] trait DumboPlatform {
1111
inline def withResourcesIn[F[_]: Sync](location: String): DumboWithResourcesPartiallyApplied[F] = {
1212
val resources = ResourceFilePath.fromResourcesDir(location)
13-
new DumboWithResourcesPartiallyApplied[F](ResourceReader.embeddedResources(Sync[F].pure(resources)))
13+
new DumboWithResourcesPartiallyApplied[F](
14+
ResourceReader.embeddedResources(
15+
readResources = Sync[F].pure(resources),
16+
locationInfo = Some(location),
17+
locationRelative = Some(location),
18+
)
19+
)
1420
}
1521
}

‎modules/core/shared/src/main/scala/dumbo/Dumbo.scala

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -222,13 +222,19 @@ class Dumbo[F[_]: Sync: Console](
222222
version = source.versionText,
223223
description = source.scriptDescription,
224224
`type` = "SQL",
225-
script = source.path.fileName,
225+
script = historyScriptPath(source),
226226
checksum = Some(source.checksum),
227227
executionTimeMs = duration.toMillis.toInt,
228228
success = true,
229229
)
230230
}
231231

232+
private def historyScriptPath(resource: ResourceFile) =
233+
resReader.locationRel match {
234+
case Some(loc) => resource.path.value.stripPrefix(s"/$loc/")
235+
case _ => resource.fileName
236+
}
237+
232238
private def validationGuard(session: Session[F], resources: ResourceFiles) =
233239
if (resources.nonEmpty) {
234240
session
@@ -446,13 +452,13 @@ class Dumbo[F[_]: Sync: Console](
446452
resources: ResourceFiles,
447453
): ValidatedNec[DumboValidationException, Unit] = {
448454
val versionedMap: Map[String, ResourceFile] = resources.versioned.map { case (v, f) => (v.text, f) }.toMap
449-
val repeatablesFileNames: Set[String] = resources.repeatable.map(_._2.fileName).toSet
455+
val repeatablesScriptNames: Set[String] = resources.repeatable.map(_._2.path.value).toSet
450456

451457
history
452458
.filter(_.`type` == "SQL")
453459
.traverse { h =>
454460
versionedMap.get(h.version.getOrElse("")) match {
455-
case None if !repeatablesFileNames.contains(h.script) =>
461+
case None if !repeatablesScriptNames.exists(_.endsWith(h.script)) =>
456462
new DumboValidationException(s"Detected applied migration not resolved locally ${h.script}")
457463
.invalidNec[Unit]
458464

‎modules/core/shared/src/main/scala/dumbo/internal/ResourcesReader.scala

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,19 @@
44

55
package dumbo.internal
66

7+
import java.io.File
8+
79
import cats.effect.Sync
810
import cats.implicits.*
911
import dumbo.ResourceFilePath
1012
import fs2.io.file.{Files as Fs2Files, Flags, Path}
1113
import fs2.{Stream, text}
1214

1315
private[dumbo] trait ResourceReader[F[_]] {
16+
// relative location info e.g. "db/migration"
17+
def locationRel: Option[String]
18+
19+
// relative or absolute location info e.g. "file:/project/resources/db/migration"
1420
def location: Option[String]
1521

1622
def list: fs2.Stream[F, ResourceFilePath]
@@ -28,12 +34,22 @@ private[dumbo] object ResourceReader {
2834

2935
@inline def absolutePath(p: Path) = if (p.isAbsolute) p else base / p
3036

37+
@scala.annotation.tailrec
38+
def listRec(dirs: List[File], files: List[File]): List[File] =
39+
dirs match {
40+
case x :: xs =>
41+
val (d, f) = x.listFiles().toList.partition(_.isDirectory())
42+
listRec(d ::: xs, f ::: files)
43+
case Nil => files
44+
}
45+
3146
new ResourceReader[F] {
32-
override val location: Option[String] = Some(absolutePath(sourceDir).toString)
47+
override val locationRel: Option[String] = Some(sourceDir.toString)
48+
override val location: Option[String] = Some(absolutePath(sourceDir).toString)
3349
override def list: Stream[F, ResourceFilePath] =
34-
Fs2Files[F]
35-
.list(absolutePath(sourceDir))
36-
.map(p => ResourceFilePath(p.toString))
50+
Stream.emits(
51+
listRec(List(new File(absolutePath(sourceDir).toString)), Nil).map(f => ResourceFilePath(f.getPath()))
52+
)
3753

3854
override def readUtf8Lines(path: ResourceFilePath): Stream[F, String] =
3955
Fs2Files[F].readUtf8Lines(absolutePath(Path(path.value)))
@@ -48,8 +64,11 @@ private[dumbo] object ResourceReader {
4864
def embeddedResources[F[_]: Sync](
4965
readResources: F[List[ResourceFilePath]],
5066
locationInfo: Option[String] = None,
67+
locationRelative: Option[String] = None,
5168
): ResourceReader[F] =
5269
new ResourceReader[F] {
70+
override val locationRel: Option[String] = locationRelative
71+
5372
override val location: Option[String] = locationInfo
5473

5574
override def list: Stream[F, ResourceFilePath] = Stream.evals(readResources)

‎modules/sample-lib/src/main/resources/sample_lib/sub_dir/V3__test_c.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE test_c();

‎modules/sample-lib/src/main/resources/sample_lib/sub_dir/sub_sub_dir/V4__test_d.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE test_d();

‎modules/test-lib/src/main/scala/TestLib.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import dumbo.ResourceFilePath
22

33
@main def run(args: String*) =
4-
val resources = ResourceFilePath.fromResourcesDir("sample_lib")
5-
val expected = List(
4+
val resources = ResourceFilePath.fromResourcesDir("sample_lib").map(_.value).toSet
5+
val expected = Set(
6+
"/sample_lib/sub_dir/sub_sub_dir/V4__test_d.sql",
7+
"/sample_lib/sub_dir/V3__test_c.sql",
68
"/sample_lib/V1__test.sql",
79
"/sample_lib/V2__test_b.sql",
810
)

‎modules/tests-flyway/src/test/scala/DumboFlywaySpec.scala

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,19 @@ trait DumboFlywaySpec extends ffstest.FTest {
138138
} yield ()
139139
}
140140

141+
dbTest("Compatible with nested directories") {
142+
val schema = "schema_1"
143+
144+
for {
145+
_ <- flywayMigrate(schema, Path("db/nested")).map(r => assert(r.migrationsExecuted == 6))
146+
historyFlyway <- loadHistory(schema)
147+
_ <- dropSchemas
148+
_ <- dumboMigrate(schema, dumboWithResources("db/nested")).map(r => assert(r.migrationsExecuted == 6))
149+
historyDumbo <- loadHistory(schema)
150+
_ = assertEqualHistory(historyDumbo, historyFlyway)
151+
} yield ()
152+
}
153+
141154
dbTest("Dumbo updates history entry of latest unsucessfully applied migration by Flyway") {
142155
// run on CockroachDb only just because it was the easiest way to reproduce a history record for an unsuccessfully applied migration with Flyway
143156
if (db == Db.CockroachDb) {

‎modules/tests/shared/src/test/non_resource/db/test_1/V2__non_resource.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE test (id SERIAL PRIMARY KEY);

‎modules/tests/shared/src/test/non_resource/db/test_1/sub/R__b.sql

Whitespace-only changes.

‎modules/tests/shared/src/test/non_resource/db/test_1/sub/V4__non_resource.sql

Whitespace-only changes.

‎modules/tests/shared/src/test/non_resource/db/test_1/sub/sub_sub/R__a.sql

Whitespace-only changes.

‎modules/tests/shared/src/test/non_resource/db/test_1/sub/sub_sub/V3__non_resource.sql

Whitespace-only changes.

‎modules/tests/shared/src/test/resources/db/nested/V1__test.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE test_a();

‎modules/tests/shared/src/test/resources/db/nested/V2__test.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE test_b();

‎modules/tests/shared/src/test/resources/db/nested/sub/R__b.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE test_f();

‎modules/tests/shared/src/test/resources/db/nested/sub/V4__test.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE test_d();

‎modules/tests/shared/src/test/resources/db/nested/sub/sub_sub/R__a.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE test_e();

‎modules/tests/shared/src/test/resources/db/nested/sub/sub_sub/V3__test.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CREATE TABLE test_c();

‎modules/tests/shared/src/test/scala/DumboResourcesSpec.scala

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,39 @@ class DumboResourcesSpec extends ffstest.FTest {
2929
} yield ()
3030
}
3131

32+
test("list migration files from resources with subirectories") {
33+
for {
34+
files <- dumboWithResources("db/nested").listMigrationFiles
35+
_ = files match {
36+
case Valid(files) =>
37+
assert(
38+
files.sorted.map(f => (f.version, f.path.fileName.toString)) == List(
39+
(ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__test.sql"),
40+
(ResourceVersion.Versioned("2", NonEmptyList.of(2)), "V2__test.sql"),
41+
(ResourceVersion.Versioned("3", NonEmptyList.of(3)), "V3__test.sql"),
42+
(ResourceVersion.Versioned("4", NonEmptyList.of(4)), "V4__test.sql"),
43+
(ResourceVersion.Repeatable("a"), "R__a.sql"),
44+
(ResourceVersion.Repeatable("b"), "R__b.sql"),
45+
)
46+
)
47+
case Invalid(errs) => fail(errs.toList.mkString("\n"))
48+
}
49+
} yield ()
50+
}
51+
3252
test("list migration files from relative path") {
3353
for {
3454
files <- Dumbo.withFilesIn[IO](Path("modules/tests/shared/src/test/non_resource/db/test_1")).listMigrationFiles
3555
_ = files match {
3656
case Valid(files) =>
3757
assert(
3858
files.sorted.map(f => (f.version, f.path.fileName.toString)) == List(
39-
(ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__non_resource.sql")
59+
(ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__non_resource.sql"),
60+
(ResourceVersion.Versioned("2", NonEmptyList.of(2)), "V2__non_resource.sql"),
61+
(ResourceVersion.Versioned("3", NonEmptyList.of(3)), "V3__non_resource.sql"),
62+
(ResourceVersion.Versioned("4", NonEmptyList.of(4)), "V4__non_resource.sql"),
63+
(ResourceVersion.Repeatable("a"), "R__a.sql"),
64+
(ResourceVersion.Repeatable("b"), "R__b.sql"),
4065
)
4166
)
4267
case Invalid(errs) => fail(errs.toList.mkString("\n"))
@@ -52,7 +77,12 @@ class DumboResourcesSpec extends ffstest.FTest {
5277
case Valid(files) =>
5378
assert(
5479
files.sorted.map(f => (f.version, f.path.fileName.toString)) == List(
55-
(ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__non_resource.sql")
80+
(ResourceVersion.Versioned("1", NonEmptyList.of(1)), "V1__non_resource.sql"),
81+
(ResourceVersion.Versioned("2", NonEmptyList.of(2)), "V2__non_resource.sql"),
82+
(ResourceVersion.Versioned("3", NonEmptyList.of(3)), "V3__non_resource.sql"),
83+
(ResourceVersion.Versioned("4", NonEmptyList.of(4)), "V4__non_resource.sql"),
84+
(ResourceVersion.Repeatable("a"), "R__a.sql"),
85+
(ResourceVersion.Repeatable("b"), "R__b.sql"),
5686
)
5787
)
5888
case Invalid(errs) => fail(errs.toList.mkString("\n"))

0 commit comments

Comments
 (0)