Skip to content

Commit aa89d19

Browse files
FoxcapadesFoxcapades
authored
Track all dependency health (#391)
1 parent 45009db commit aa89d19

File tree

32 files changed

+348
-53
lines changed

32 files changed

+348
-53
lines changed

‎Dockerfile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ ARG GITHUB_TOKEN
55

66
WORKDIR /workspace
77

8-
COPY settings.gradle.kts settings.gradle.kts
9-
COPY build.gradle.kts build.gradle.kts
8+
COPY ["settings.gradle.kts", "build.gradle.kts", "./"]
109
COPY gradle/libs.versions.toml gradle/libs.versions.toml
10+
11+
RUN gradle --no-daemon download-dependencies
12+
1113
COPY service service
1214
COPY lib lib
1315

‎build.gradle.kts

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import org.gradle.api.tasks.testing.logging.TestLogEvent
33
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
44

55
plugins {
6-
kotlin("jvm") version "2.1.10"
6+
kotlin("jvm") version libs.versions.kotlin
77
id("com.github.johnrengelman.shadow") version "8.1.1"
88
}
99

@@ -65,7 +65,7 @@ tasks.shadowJar {
6565
}
6666
}
6767

68-
tasks.create("compile-design-doc") {
68+
tasks.register("compile-design-doc") {
6969
doLast {
7070
val command = arrayOf(
7171
"asciidoctor",
@@ -86,7 +86,7 @@ tasks.create("compile-design-doc") {
8686
}
8787
}
8888

89-
tasks.create("generate-raml-docs") {
89+
tasks.register("generate-raml-docs") {
9090
dependsOn(":service:rest-service:generate-raml-docs")
9191

9292
doLast {
@@ -107,6 +107,20 @@ tasks.create("generate-raml-docs") {
107107
}
108108
}
109109

110+
tasks.register("download-dependencies") {
111+
configurations {
112+
create("download") {
113+
project.versionCatalogs.forEach { catalog ->
114+
catalog.libraryAliases.forEach { libName ->
115+
dependencies.addLater(catalog.findLibrary(libName).get())
116+
}
117+
}
118+
119+
this.files
120+
}
121+
}
122+
}
123+
110124
dependencies {
111125
implementation(project(":service:bootstrap"))
112126
}

‎gradle/libs.versions.toml

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
1+
[versions]
2+
3+
kotlin = "2.1.10"
4+
15
[libraries]
26

37
# VPDB
48
fgputil-db = { group = "org.gusdb", name = "fgputil-db", version = "2.16.0-jakarta" }
59

6-
container-core = { group = "org.veupathdb.lib", name = "jaxrs-container-core", version = "9.1.3" }
10+
container-core = { group = "org.veupathdb.lib", name = "jaxrs-container-core", version = "9.4.2" }
711
container-multipart = { group = "org.veupathdb.lib", name = "multipart-jackson-pojo", version = "1.1.7" }
812

913
vdi-common = { group = "org.veupathdb.vdi", name = "vdi-component-common", version = "17.1.6" }
@@ -42,6 +46,9 @@ prometheus-common = { group = "io.prometheus", name = "simpleclient_common", ver
4246
# S3
4347
s34k = { group = "org.veupathdb.lib.s3", name = "s34k-minio", version = "0.7.2+" }
4448

49+
# Utilities
50+
deque = { group = "io.k-libs", name = "deque", version = "0.9.0" }
51+
4552
#
4653
# Unit Testing
4754
#

‎lib/app-db/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ plugins {
44

55
dependencies {
66
implementation(project(":lib:env"))
7+
implementation(project(":lib:health"))
78
implementation(project(":lib:ldap"))
89

910
implementation(libs.vdi.common)

‎lib/app-db/src/main/kotlin/vdi/component/db/app/AppDatabaseRegistry.kt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@ import org.veupathdb.vdi.lib.common.env.Environment
1010
import org.veupathdb.vdi.lib.common.field.DataType
1111
import org.veupathdb.vdi.lib.common.field.ProjectID
1212
import org.veupathdb.vdi.lib.common.field.SecretString
13+
import vdi.component.db.app.health.DatabaseDependency
1314
import vdi.component.env.EnvKey
1415
import vdi.component.ldap.LDAP
16+
import vdi.health.RemoteDependencies
1517
import javax.sql.DataSource
1618

1719
@Suppress("NOTHING_TO_INLINE")
@@ -196,6 +198,8 @@ object AppDatabaseRegistry {
196198
}
197199
}
198200
}
201+
202+
RemoteDependencies.register(DatabaseDependency(entry))
199203
}
200204
}
201205
}

‎lib/app-db/src/main/kotlin/vdi/component/db/app/health/DatabaseDependency.kt

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package vdi.component.db.app.health
2+
3+
import org.slf4j.LoggerFactory
4+
import vdi.component.db.app.AppDBPlatform
5+
import vdi.component.db.app.AppDBRegistryEntry
6+
import vdi.health.Dependency
7+
import vdi.health.StaticDependency
8+
9+
class DatabaseDependency(
10+
private val connection: AppDBRegistryEntry,
11+
extra: Map<String, Any> = emptyMap(),
12+
) : StaticDependency(connection.name, "", connection.host, connection.port, extra.appendDbInfo(connection)) {
13+
override fun checkStatus() =
14+
when (connection.platform) {
15+
AppDBPlatform.Postgres -> pgStatus()
16+
AppDBPlatform.Oracle -> oraStatus()
17+
}
18+
19+
private fun oraStatus() = runQuery("SELECT 1 FROM dual")
20+
21+
private fun pgStatus() = runQuery("SELECT 1")
22+
23+
private fun runQuery(query: String) =
24+
try {
25+
connection.source.connection.use { c -> c.createStatement().use { s -> s.execute(query) } }
26+
Dependency.Status.Ok
27+
} catch (e: Throwable) {
28+
LoggerFactory.getLogger(javaClass).error("postgres database healthcheck failed", e)
29+
Dependency.Status.NotOk;
30+
}
31+
}
32+
33+
private fun Map<String, Any>.appendDbInfo(entry: AppDBRegistryEntry): Map<String, Any> {
34+
val map = if (this is MutableMap)
35+
this
36+
else
37+
this.toMutableMap()
38+
39+
map["host"] = entry.host
40+
map["controlSchema"] = entry.ctlSchema
41+
map["dataSchema"] = entry.dataSchema
42+
43+
return map
44+
}

‎lib/cache-db/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ plugins {
44

55
dependencies {
66
implementation(project(":lib:env"))
7+
implementation(project(":lib:health"))
78

89
implementation(libs.vdi.json)
910
implementation(libs.vdi.common)

‎lib/cache-db/src/main/kotlin/vdi/component/db/cache/CacheDBImpl.kt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,13 @@ import org.veupathdb.vdi.lib.common.env.optUShort
88
import org.veupathdb.vdi.lib.common.env.require
99
import org.veupathdb.vdi.lib.common.field.DatasetID
1010
import org.veupathdb.vdi.lib.common.field.UserID
11+
import vdi.component.db.cache.health.DatabaseDependency
1112
import vdi.component.db.cache.model.BrokenImportListQuery
1213
import vdi.component.db.cache.model.DatasetListQuery
1314
import vdi.component.db.cache.query.AdminAllDatasetsQuery
1415
import vdi.component.db.cache.sql.select.*
1516
import vdi.component.env.EnvKey
17+
import vdi.health.RemoteDependencies
1618
import javax.sql.DataSource
1719
import org.postgresql.Driver as PostgresDriver
1820

@@ -50,6 +52,8 @@ internal object CacheDBImpl: CacheDB {
5052

5153
dataSource = HikariDataSource(config)
5254
details = CacheDBConnectionDetails(host, port, name)
55+
56+
RemoteDependencies.register(DatabaseDependency(this))
5357
}
5458

5559
override fun selectDataset(datasetID: DatasetID) = connection.use { it.selectDataset(datasetID) }

‎lib/cache-db/src/main/kotlin/vdi/component/db/cache/health/DatabaseDependency.kt

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package vdi.component.db.cache.health
2+
3+
import org.slf4j.LoggerFactory
4+
import vdi.component.db.cache.CacheDBImpl
5+
import vdi.health.Dependency
6+
import vdi.health.StaticDependency
7+
8+
internal class DatabaseDependency(private val connection: CacheDBImpl)
9+
: StaticDependency("Internal Cache DB", "", connection.details.host, connection.details.port)
10+
{
11+
override fun checkStatus() =
12+
try {
13+
connection.dataSource.connection.use { c -> c.createStatement().use { s -> s.execute("SELECT 1") } }
14+
Dependency.Status.Ok
15+
} catch (e: Throwable) {
16+
LoggerFactory.getLogger(javaClass).error("postgres database healthcheck failed", e)
17+
Dependency.Status.NotOk;
18+
}
19+
}

‎lib/health/build.gradle.kts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
plugins {
2+
kotlin("jvm")
3+
}
4+
5+
dependencies {
6+
implementation(libs.deque)
7+
}

‎lib/health/src/main/kotlin/vdi/health/Dependency.kt

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package vdi.health
2+
3+
interface Dependency {
4+
val name: String
5+
val protocol: String
6+
val host: String
7+
val port: UShort
8+
val extra: Map<String, Any>
9+
10+
enum class Status {
11+
Ok,
12+
NotOk,
13+
Unknown,
14+
}
15+
16+
fun checkStatus(): Status
17+
18+
fun urlString() =
19+
if (protocol.isNotBlank())
20+
"$protocol://$host"
21+
else
22+
host
23+
}

‎lib/health/src/main/kotlin/vdi/health/DynamicDependency.kt

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package vdi.health
2+
3+
open class DynamicDependency(
4+
override val name: String,
5+
override val protocol: String,
6+
override val host: String,
7+
override val port: UShort,
8+
val extraSupplier: () -> Map<String, Any>
9+
) : Dependency {
10+
override val extra: Map<String, Any>
11+
get() = extraSupplier()
12+
13+
override fun checkStatus() = Dependency.Status.Ok
14+
15+
override fun toString() =
16+
if (protocol.isNotBlank())
17+
"$protocol://$host:$port"
18+
else
19+
"$host:$port"
20+
}

‎lib/health/src/main/kotlin/vdi/health/RemoteDependencies.kt

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package vdi.health
2+
3+
import io.klibs.collections.Deque
4+
5+
object RemoteDependencies : Iterable<Dependency> {
6+
private val registry = Deque<Dependency>()
7+
8+
fun register(dependency: Dependency) {
9+
registry.append(dependency)
10+
}
11+
12+
fun register(
13+
name: String,
14+
host: String,
15+
port: UShort,
16+
protocol: String = "",
17+
extraProvider: (() -> Map<String, Any>)? = null,
18+
) {
19+
if (extraProvider != null)
20+
registry.append(DynamicDependency(name, protocol, host, port, extraProvider))
21+
else
22+
registry.append(StaticDependency(name, protocol, host, port))
23+
}
24+
25+
override fun iterator(): Iterator<Dependency> {
26+
return object : Iterator<Dependency> {
27+
private val raw = registry.iterator()
28+
override fun hasNext() = raw.hasNext()
29+
override fun next() = raw.next()
30+
}
31+
}
32+
}

‎lib/health/src/main/kotlin/vdi/health/StaticDependency.kt

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package vdi.health
2+
3+
open class StaticDependency(
4+
override val name: String,
5+
override val protocol: String,
6+
override val host: String,
7+
override val port: UShort,
8+
override val extra: Map<String, Any> = emptyMap()
9+
) : Dependency {
10+
override fun toString() =
11+
if (protocol.isNotBlank())
12+
"$protocol://$host:$port"
13+
else
14+
"$host:$port"
15+
16+
override fun checkStatus() = Dependency.Status.Ok
17+
}

‎lib/kafka/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ plugins {
44

55
dependencies {
66
implementation(project(":lib:env"))
7+
implementation(project(":lib:health"))
78

89
implementation(libs.vdi.json)
910
implementation(libs.vdi.common)

‎lib/kafka/src/main/kotlin/vdi/component/kafka/index.kt

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,27 @@ import org.apache.kafka.clients.consumer.ConsumerConfig
44
import org.apache.kafka.clients.producer.ProducerConfig
55
import org.apache.kafka.common.serialization.StringDeserializer
66
import org.apache.kafka.common.serialization.StringSerializer
7+
import org.veupathdb.vdi.lib.common.util.HostAddress
8+
import vdi.health.RemoteDependencies
79
import java.util.*
10+
import kotlin.collections.HashSet
811
import org.apache.kafka.clients.consumer.KafkaConsumer as KConsumer
912
import org.apache.kafka.clients.producer.KafkaProducer as KProducer
1013

14+
private val knownServers = HashSet<HostAddress>(1)
15+
private fun init(servers: Iterable<HostAddress>) {
16+
synchronized(knownServers) {
17+
for (server in servers) {
18+
if (server !in knownServers) {
19+
knownServers.add(server)
20+
RemoteDependencies.register("Kafka ${server}", server.host, server.port)
21+
}
22+
}
23+
}
24+
}
25+
1126
fun KafkaConsumer(topic: String, config: KafkaConsumerConfig): KafkaConsumer {
27+
init(config.servers)
1228
val props = Properties()
1329
.apply {
1430
setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.servers.joinToString(",") { it.toString() })
@@ -43,6 +59,7 @@ fun KafkaConsumer(topic: String, config: KafkaConsumerConfig): KafkaConsumer {
4359
}
4460

4561
fun KafkaProducer(config: KafkaProducerConfig): KafkaProducer {
62+
init(config.servers)
4663
val props = Properties()
4764
.apply {
4865
setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.servers.joinToString(",") { it.toString() })
@@ -65,4 +82,4 @@ fun KafkaProducer(config: KafkaProducerConfig): KafkaProducer {
6582
}
6683

6784
return KafkaProducerImpl(StringSerializer().let { KProducer(props, it, it) })
68-
}
85+
}

‎lib/ldap/build.gradle.kts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,11 @@ plugins {
33
}
44

55
dependencies {
6-
implementation(libs.vdi.common)
6+
implementation(project(":lib:health"))
7+
78
api(libs.ldap)
9+
10+
implementation(libs.vdi.common)
811
implementation(libs.log.slf4j)
912

1013
testImplementation(kotlin("test"))

‎lib/ldap/src/main/kotlin/vdi/component/ldap/LDAP.kt

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import org.veupathdb.vdi.lib.common.env.EnvKey
77
import org.veupathdb.vdi.lib.common.env.Environment
88
import org.veupathdb.vdi.lib.common.env.reqHostAddresses
99
import org.veupathdb.vdi.lib.common.env.require
10+
import vdi.health.RemoteDependencies
1011

1112
object LDAP {
1213

@@ -28,5 +29,9 @@ object LDAP {
2829
val oracleDN = env.require(EnvKey.LDAP.OracleBaseDN)
2930

3031
ldap = LDAP(LDAPConfig(servers, oracleDN))
32+
33+
servers.forEach {
34+
RemoteDependencies.register("LDAP ${it.host}", it.host, it.port)
35+
}
3136
}
32-
}
37+
}

0 commit comments

Comments
 (0)