NoSQL persistence

Author: snazy

LICENSE

@@ -304,6 +304,9 @@ This product includes code from Project Nessie.
 * tools/config-docs/generator/src/test/java/tests/smallrye/SomeEnum.java
 * tools/config-docs/generator/src/test/java/tests/smallrye/VeryNested.java
 
+Code underneath the components/persistence directory, especially pluggable object types, index related, cache,
+atomic commit logic and fundamental persistence implementations.
+
 Copyright: Copyright 2015-2025 Dremio Corporation
 Home page: https://projectnessie.org/
 License: https://www.apache.org/licenses/LICENSE-2.0

bom/build.gradle.kts

@@ -43,6 +43,40 @@ dependencies {
     api(project(":polaris-idgen-impl"))
     api(project(":polaris-idgen-spi"))
 
+    api(project(":polaris-nodes-api"))
+    api(project(":polaris-nodes-impl"))
+    api(project(":polaris-nodes-spi"))
+    api(project(":polaris-nodes-store"))
+
+    api(project(":polaris-realms-api"))
+    api(project(":polaris-realms-impl"))
+    api(project(":polaris-realms-spi"))
+    api(project(":polaris-realms-store"))
+
+    api(project(":polaris-authz-api"))
+    api(project(":polaris-authz-impl"))
+    api(project(":polaris-authz-spi"))
+    api(project(":polaris-authz-store"))
+
+    api(project(":polaris-persistence-api"))
+    api(project(":polaris-persistence-impl"))
+    api(project(":polaris-persistence-benchmark"))
+    api(project(":polaris-persistence-bridge"))
+    api(project(":polaris-persistence-cache"))
+    api(project(":polaris-persistence-cdi-common"))
+    api(project(":polaris-persistence-cdi-quarkus"))
+    api(project(":polaris-persistence-cdi-weld"))
+    api(project(":polaris-persistence-commits"))
+    api(project(":polaris-persistence-correctness"))
+    api(project(":polaris-persistence-delegate"))
+    api(project(":polaris-persistence-index"))
+    api(project(":polaris-persistence-standalone"))
+    api(project(":polaris-persistence-testextension"))
+    api(project(":polaris-persistence-types"))
+
+    api(project(":polaris-persistence-inmemory"))
+    api(project(":polaris-persistence-mongodb"))
+
     api(project(":polaris-config-docs-annotations"))
     api(project(":polaris-config-docs-generator"))
 

build.gradle.kts

@@ -72,6 +72,10 @@ tasks.named<RatTask>("rat").configure {
   // Manifest files do not allow comments
   excludes.add("tools/version/src/jarTest/resources/META-INF/FAKE_MANIFEST.MF")
 
+  excludes.add(
+    "nosql/persistence/index/src/testFixtures/resources/org/apache/polaris/persistence/indexes/words.gz"
+  )
+
   excludes.add("ide-name.txt")
   excludes.add("version.txt")
   excludes.add(".git")

gradle/libs.versions.toml

@@ -37,6 +37,7 @@ swagger = "1.6.15"
 # If a dependency is removed, check whether the LICENSE and/or NOTICE files need to be adopted
 # (aka mention of the dependency removed).
 #
+agrona = { module = "org.agrona:agrona", version = "2.1.0" }
 antlr4-runtime = { module = "org.antlr:antlr4-runtime", version.strictly = "4.9.3" } # spark integration tests
 assertj-core = { module = "org.assertj:assertj-core", version = "3.27.3" }
 auth0-jwt = { module = "com.auth0:java-jwt", version = "4.5.0" }
@@ -47,6 +48,7 @@ caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.2.0
 commons-codec1 = { module = "commons-codec:commons-codec", version = "1.18.0" }
 commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.17.0" }
 commons-text = { module = "org.apache.commons:commons-text", version = "1.13.0" }
+docker-java-api = { module = "com.github.docker-java:docker-java-api", version = "3.5.0" }
 eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.5" }
 errorprone = { module = "com.google.errorprone:error_prone_core", version = "2.37.0" }
 google-cloud-storage-bom = { module = "com.google.cloud:google-cloud-storage-bom", version = "2.50.0" }
@@ -79,7 +81,9 @@ logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18
 micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.14.5" }
 mockito-core = { module = "org.mockito:mockito-core", version = "5.17.0" }
 mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.17.0" }
+mongodb-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version = "5.4.0" }
 opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "1.49.0" }
+opentelemetry-instrumentation-bom-alpha = { module = "io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom-alpha", version= "2.14.0-alpha" }
 opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version = "1.32.0" }
 picocli = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }
 picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }
@@ -92,6 +96,7 @@ s3mock-testcontainers = { module = "com.adobe.testing:s3mock-testcontainers", ve
 slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
 smallrye-common-annotation = { module = "io.smallrye.common:smallrye-common-annotation", version = "2.11.0" }
 smallrye-config-core = { module = "io.smallrye.config:smallrye-config-core", version = "3.12.4" }
+smallrye-jandex = { module = "io.smallrye:jandex", version = "3.2.7" }
 spark35-sql-scala212 = { module = "org.apache.spark:spark-sql_2.12", version.ref = "spark35" }
 spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version = "4.9.3" }
 swagger-annotations = { module = "io.swagger:swagger-annotations", version.ref = "swagger" }

gradle/projects.main.properties

@@ -39,7 +39,6 @@ polaris-immutables=tools/immutables
 polaris-container-spec-helper=tools/container-spec-helper
 polaris-version=tools/version
 polaris-misc-types=tools/misc-types
-polaris-persistence-varint=nosql/persistence/varint
 
 polaris-config-docs-annotations=tools/config-docs/annotations
 polaris-config-docs-generator=tools/config-docs/generator
@@ -52,4 +51,40 @@ polaris-async-vertx=nosql/async/vertx
 # id generation
 polaris-idgen-api=nosql/idgen/api
 polaris-idgen-impl=nosql/idgen/impl
+polaris-idgen-mocks=nosql/idgen/mocks
 polaris-idgen-spi=nosql/idgen/spi
+# nodes
+polaris-nodes-api=nosql/nodes/api
+polaris-nodes-impl=nosql/nodes/impl
+polaris-nodes-spi=nosql/nodes/spi
+polaris-nodes-store=nosql/nodes/store
+# realms
+polaris-realms-api=nosql/realms/api
+polaris-realms-impl=nosql/realms/impl
+polaris-realms-spi=nosql/realms/spi
+polaris-realms-store=nosql/realms/store
+# authz
+polaris-authz-api=nosql/authz/api
+polaris-authz-impl=nosql/authz/impl
+polaris-authz-spi=nosql/authz/spi
+polaris-authz-store=nosql/authz/store
+# persistence / database agnostic
+polaris-persistence-api=nosql/persistence/api
+polaris-persistence-impl=nosql/persistence/impl
+polaris-persistence-benchmark=nosql/persistence/benchmark
+polaris-persistence-bridge=nosql/persistence/bridge
+polaris-persistence-cache=nosql/persistence/cache
+polaris-persistence-cdi-common=nosql/persistence/cdi/common
+polaris-persistence-cdi-quarkus=nosql/persistence/cdi/quarkus
+polaris-persistence-cdi-weld=nosql/persistence/cdi/weld
+polaris-persistence-commits=nosql/persistence/commits
+polaris-persistence-correctness=nosql/persistence/correctness
+polaris-persistence-delegate=nosql/persistence/delegate
+polaris-persistence-index=nosql/persistence/index
+polaris-persistence-standalone=nosql/persistence/standalone
+polaris-persistence-testextension=nosql/persistence/testextension
+polaris-persistence-types=nosql/persistence/types
+polaris-persistence-varint=nosql/persistence/varint
+# persistence / database specific implementations
+polaris-persistence-inmemory=nosql/persistence/db/inmemory
+polaris-persistence-mongodb=nosql/persistence/db/mongodb

nosql/async/java/src/main/java/org/apache/polaris/async/java/JavaPoolAsyncExec.java

@@ -51,7 +51,7 @@
  * Java thread pool to execute blocking task.
  */
 @ApplicationScoped
-class JavaPoolAsyncExec implements AsyncExec {
+public class JavaPoolAsyncExec implements AsyncExec {
   private static final Logger LOGGER = LoggerFactory.getLogger(JavaPoolAsyncExec.class.getName());
 
   private final ThreadPoolExecutor executorService;

nosql/authz/README.md

@@ -0,0 +1,64 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# AuthZ framework with pluggable privileges
+
+Provides a framework and implementations pluggable privileges and privilege checks.
+
+## Privileges
+
+A privilege is globally identified by its name. Privileges can be inheritable (from its parents) or not. Multiple
+privileges can be grouped together to a _composite_ privilege (think: `ALL_DML` having `SELECT`, `INSERT`, `UPDATE` and
+`DELETE`) - a composite privilege matches, if all its individual privileges match. Multiple privileges can also be
+grouped to an _alternative_ privilege, which matches if any of its individual privileges matches.
+
+Available privileges are provided by one or more `PrivilegeProvider`s, which are discovered at runtime.
+Note: currently there is only one `ProvilegeProvider` that plugs in the Polaris privileges.
+
+## ACLs, ACL entries and ACL chains
+
+Each securable object can have its own ACL. ACLs consist of ACL entries, which define the _granted_ and _restricted_
+privileges by role name. The the number of roles is technically unbounded and the number of ACL entries can become
+quite large.
+
+This framework implements [separation of duties](https://en.wikipedia.org/wiki/Separation_of_duties) ("SoD"), which is a
+quite demanded functionality not just by large(r) user organizations. TL;DR _SoD_ allows "security administrators" to
+grant and revoke privileges to other users, but not leverage those privileges themselves.
+
+The _effective_ set of privileges for a specific operation performed by a specific caller needs to be computed against
+the target objects and their parents. _ACL chains_ are the vehicle to model this hierarchy and let the implementation
+compute the set of _effective_ privileges based on the individual ACLs and roles.
+
+Note: Privilege checks and _SoD_ are currently not performed via this framework.
+
+## Jackson support & Storage friendly representation
+
+The persistable types `Acl`, `AclEntry`, and `PrivilegeSet` can all be serialized using Jackson.
+
+As the number of ACL entries can become quite large, space efficient serialization is quite important. The
+implementation uses bit-set encoding when serializing `PrivilegeSet`s for persistence.
+
+## Code structure
+
+The code is structured into multiple modules. Consuming code should almost always pull in only the API module.
+
+* `polaris-authz-api` provides the necessary Java interfaces and immutable types.
+* `polaris-authz-impl` provides the storage agnostic implementation.
+* `polaris-authz-spi` provides the necessary interfaces to provide custom privileges and storage implementation.
+* `polaris-authz-store` provides the storage implementation based on `polaris-persistence-api`.

nosql/authz/api/build.gradle.kts

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  alias(libs.plugins.jandex)
+  id("polaris-server")
+}
+
+description = "Polaris AuthZ API"
+
+dependencies {
+  implementation(libs.guava)
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+
+  compileOnly(libs.jakarta.annotation.api)
+  compileOnly(libs.jakarta.validation.api)
+  compileOnly(libs.jakarta.inject.api)
+  compileOnly(libs.jakarta.enterprise.cdi.api)
+
+  compileOnly(project(":polaris-immutables"))
+  annotationProcessor(project(":polaris-immutables", configuration = "processor"))
+}

nosql/authz/api/src/main/java/org/apache/polaris/authz/api/Acl.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.authz.api;
+
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import jakarta.annotation.Nonnull;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+public interface Acl {
+
+  void entriesForRoleIds(
+      @Nonnull Set<String> roleIds, @Nonnull Consumer<AclEntry> aclEntryConsumer);
+
+  void forEach(@Nonnull BiConsumer<String, AclEntry> consumer);
+
+  interface AclBuilder {
+    @CanIgnoreReturnValue
+    AclBuilder from(@Nonnull Acl instance);
+
+    @CanIgnoreReturnValue
+    AclBuilder addEntry(@Nonnull String roleId, @Nonnull AclEntry entry);
+
+    @CanIgnoreReturnValue
+    AclBuilder removeEntry(@Nonnull String roleId);
+
+    /**
+     * Add, remove or update an {@linkplain AclEntry ACL entry} for a role.
+     *
+     * <p>The {@linkplain Consumer consumer} is called with an empty builder, if no ACL entry for
+     * the role exists, otherwise with a builder constructed from the existing entry. If the given
+     * {@linkplain Consumer consumer} removes all privileges from the ACL entry, the ACL entry will
+     * be removed.
+     */
+    @CanIgnoreReturnValue
+    AclBuilder modify(@Nonnull String roleId, @Nonnull Consumer<AclEntry.AclEntryBuilder> entry);
+
+    Acl build();
+  }
+}

nosql/authz/api/src/main/java/org/apache/polaris/authz/api/AclChain.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.authz.api;
+
+import java.util.Optional;
+import org.apache.polaris.immutables.PolarisImmutable;
+import org.immutables.value.Value;
+
+/** Container for an {@linkplain Acl ACL} of an individual entity and a pointer to its parent. */
+@PolarisImmutable
+public interface AclChain {
+  @Value.Parameter(order = 1)
+  Acl acl();
+
+  @Value.Parameter(order = 2)
+  Optional<AclChain> parent();
+
+  static AclChain aclChain(Acl acl, Optional<AclChain> parent) {
+    return ImmutableAclChain.of(acl, parent);
+  }
+}