Skip to content

[DRAFT] NoSQL database agnostic persistence #1189

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 10 commits into
base: main
Choose a base branch
from
+48,220 −164
Draft

[DRAFT] NoSQL database agnostic persistence #1189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
e0d1086
Auto-bootstrap: add verbose logging
snazy Apr 9, 2025
c26c19b
PolarisRestCatalogIntegrationTest: Always purge generic tables
snazy Apr 24, 2025
f277b13
Spark-IT: use correct configurations
snazy Apr 24, 2025
13107e0
Add jcstress dependencies
snazy Mar 26, 2025
d8d471d
Add jmh dependencies
snazy Apr 4, 2025
56c5c90
Add component to abstract async execution
snazy Mar 17, 2025
51b2721
Unique ID generation
snazy Mar 17, 2025
f77347d
NoSQL persistence
snazy Mar 13, 2025
5e3c5aa
NoSQL: Adopt to "Make entity lookups by id honor the specified entity…
snazy Apr 24, 2025
5c5b261
NoSQL: Filter on correct subtype
snazy Apr 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
3 changes: 3 additions & 0 deletions LICENSE
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,9 @@ This product includes code from Project Nessie.
* tools/config-docs/generator/src/test/java/tests/smallrye/SomeEnum.java
* tools/config-docs/generator/src/test/java/tests/smallrye/VeryNested.java

Code underneath the components/persistence directory, especially pluggable object types, index related, cache,
atomic commit logic and fundamental persistence implementations.

Copyright: Copyright 2015-2025 Dremio Corporation
Home page: https://projectnessie.org/
License: https://www.apache.org/licenses/LICENSE-2.0
42 changes: 42 additions & 0 deletions bom/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,48 @@ dependencies {
api(project(":polaris-version"))
api(project(":polaris-persistence-varint"))

api(project(":polaris-async-api"))
api(project(":polaris-async-java"))
api(project(":polaris-async-vertx"))

api(project(":polaris-idgen-api"))
api(project(":polaris-idgen-impl"))
api(project(":polaris-idgen-spi"))

api(project(":polaris-nodes-api"))
api(project(":polaris-nodes-impl"))
api(project(":polaris-nodes-spi"))
api(project(":polaris-nodes-store"))

api(project(":polaris-realms-api"))
api(project(":polaris-realms-impl"))
api(project(":polaris-realms-spi"))
api(project(":polaris-realms-store"))

api(project(":polaris-authz-api"))
api(project(":polaris-authz-impl"))
api(project(":polaris-authz-spi"))
api(project(":polaris-authz-store"))

api(project(":polaris-persistence-api"))
api(project(":polaris-persistence-impl"))
api(project(":polaris-persistence-benchmark"))
api(project(":polaris-persistence-bridge"))
api(project(":polaris-persistence-cache"))
api(project(":polaris-persistence-cdi-common"))
api(project(":polaris-persistence-cdi-quarkus"))
api(project(":polaris-persistence-cdi-weld"))
api(project(":polaris-persistence-commits"))
api(project(":polaris-persistence-correctness"))
api(project(":polaris-persistence-delegate"))
api(project(":polaris-persistence-index"))
api(project(":polaris-persistence-standalone"))
api(project(":polaris-persistence-testextension"))
api(project(":polaris-persistence-types"))

api(project(":polaris-persistence-inmemory"))
api(project(":polaris-persistence-mongodb"))

api(project(":polaris-config-docs-annotations"))
api(project(":polaris-config-docs-generator"))

Expand Down
4 changes: 4 additions & 0 deletions build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ tasks.named<RatTask>("rat").configure {
// Manifest files do not allow comments
excludes.add("tools/version/src/jarTest/resources/META-INF/FAKE_MANIFEST.MF")

excludes.add(
"nosql/persistence/index/src/testFixtures/resources/org/apache/polaris/persistence/indexes/words.gz"
)

excludes.add("ide-name.txt")
excludes.add("version.txt")
excludes.add(".git")
Expand Down
14 changes: 14 additions & 0 deletions gradle/libs.versions.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ hadoop = "3.4.1"
iceberg = "1.8.1" # Ensure to update the iceberg version in regtests to keep regtests up-to-date
quarkus = "3.21.2"
immutables = "2.10.1"
jmh = "1.37"
picocli = "4.7.7"
scala212 = "2.12.19"
spark35 = "3.5.5"
Expand All @@ -36,6 +37,7 @@ swagger = "1.6.15"
# If a dependency is removed, check whether the LICENSE and/or NOTICE files need to be adopted
# (aka mention of the dependency removed).
#
agrona = { module = "org.agrona:agrona", version = "2.1.0" }
antlr4-runtime = { module = "org.antlr:antlr4-runtime", version.strictly = "4.9.3" } # spark integration tests
assertj-core = { module = "org.assertj:assertj-core", version = "3.27.3" }
auth0-jwt = { module = "com.auth0:java-jwt", version = "4.5.0" }
Expand All @@ -46,6 +48,7 @@ caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.2.0
commons-codec1 = { module = "commons-codec:commons-codec", version = "1.18.0" }
commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0" }
commons-text = { module = "org.apache.commons:commons-text", version = "1.13.1" }
docker-java-api = { module = "com.github.docker-java:docker-java-api", version = "3.5.0" }
eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.6" }
errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.38.0" }
google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.51.0" }
Expand All @@ -70,12 +73,17 @@ jakarta-servlet-api = { module = "jakarta.servlet:jakarta.servlet-api", version
jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", version = "3.1.1" }
jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0.0" }
javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" }
jcstress-core = { module = "org.openjdk.jcstress:jcstress-core", version = "0.16" }
jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" }
jmh-generator-annprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" }
junit-bom = { module = "org.junit:junit-bom", version = "5.12.2" }
logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18" }
micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.14.6" }
mockito-core = { module = "org.mockito:mockito-core", version = "5.17.0" }
mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.17.0" }
mongodb-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version = "5.4.0" }
opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "1.49.0" }
opentelemetry-instrumentation-bom-alpha = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha", version= "2.14.0-alpha" }
opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version = "1.32.0" }
picocli = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }
picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }
Expand All @@ -88,15 +96,21 @@ s3mock-testcontainers = { module = "com.adobe.testing:s3mock-testcontainers", ve
slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
smallrye-common-annotation = { module = "io.smallrye.common:smallrye-common-annotation", version = "2.12.0" }
smallrye-config-core = { module = "io.smallrye.config:smallrye-config-core", version = "3.12.4" }
smallrye-jandex = { module = "io.smallrye:jandex", version = "3.2.7" }
spark35-sql-scala212 = { module = "org.apache.spark:spark-sql_2.12", version.ref = "spark35" }
spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version = "4.9.3" }
swagger-annotations = { module = "io.swagger:swagger-annotations", version.ref = "swagger" }
swagger-jaxrs = { module = "io.swagger:swagger-jaxrs", version.ref = "swagger" }
testcontainers-bom = { module = "org.testcontainers:testcontainers-bom", version = "1.21.0" }
threeten-extra = { module = "org.threeten:threeten-extra", version = "1.8.0" }
vertx-core = { module = "io.vertx:vertx-core", version = "4.5.14" }
weld-se-core = { module = "org.jboss.weld.se:weld-se-core", version = "6.0.2.Final" }
weld-junit5 = { module = "org.jboss.weld:weld-junit5", version = "4.0.4.Final" }

[plugins]
jandex = { id = "org.kordamp.gradle.jandex", version = "2.1.0" }
jcstress = { id = "io.github.reyerizo.gradle.jcstress", version = "0.8.15" }
jmh = { id = "me.champeau.jmh", version = "0.7.3" }
openapi-generator = { id = "org.openapi.generator", version = "7.12.0" }
quarkus = { id = "io.quarkus", version.ref = "quarkus" }
rat = { id = "org.nosphere.apache.rat", version = "0.8.1" }
46 changes: 45 additions & 1 deletion gradle/projects.main.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,52 @@ polaris-immutables=tools/immutables
polaris-container-spec-helper=tools/container-spec-helper
polaris-version=tools/version
polaris-misc-types=tools/misc-types
polaris-persistence-varint=nosql/persistence/varint

polaris-config-docs-annotations=tools/config-docs/annotations
polaris-config-docs-generator=tools/config-docs/generator
polaris-config-docs-site=tools/config-docs/site

# executor abstraction
polaris-async-api=nosql/async/api
polaris-async-java=nosql/async/java
polaris-async-vertx=nosql/async/vertx
# id generation
polaris-idgen-api=nosql/idgen/api
polaris-idgen-impl=nosql/idgen/impl
polaris-idgen-mocks=nosql/idgen/mocks
polaris-idgen-spi=nosql/idgen/spi
# nodes
polaris-nodes-api=nosql/nodes/api
polaris-nodes-impl=nosql/nodes/impl
polaris-nodes-spi=nosql/nodes/spi
polaris-nodes-store=nosql/nodes/store
# realms
polaris-realms-api=nosql/realms/api
polaris-realms-impl=nosql/realms/impl
polaris-realms-spi=nosql/realms/spi
polaris-realms-store=nosql/realms/store
# authz
polaris-authz-api=nosql/authz/api
polaris-authz-impl=nosql/authz/impl
polaris-authz-spi=nosql/authz/spi
polaris-authz-store=nosql/authz/store
# persistence / database agnostic
polaris-persistence-api=nosql/persistence/api
polaris-persistence-impl=nosql/persistence/impl
polaris-persistence-benchmark=nosql/persistence/benchmark
polaris-persistence-bridge=nosql/persistence/bridge
polaris-persistence-cache=nosql/persistence/cache
polaris-persistence-cdi-common=nosql/persistence/cdi/common
polaris-persistence-cdi-quarkus=nosql/persistence/cdi/quarkus
polaris-persistence-cdi-weld=nosql/persistence/cdi/weld
polaris-persistence-commits=nosql/persistence/commits
polaris-persistence-correctness=nosql/persistence/correctness
polaris-persistence-delegate=nosql/persistence/delegate
polaris-persistence-index=nosql/persistence/index
polaris-persistence-standalone=nosql/persistence/standalone
polaris-persistence-testextension=nosql/persistence/testextension
polaris-persistence-types=nosql/persistence/types
polaris-persistence-varint=nosql/persistence/varint
# persistence / database specific implementations
polaris-persistence-inmemory=nosql/persistence/db/inmemory
polaris-persistence-mongodb=nosql/persistence/db/mongodb
Loading