NoSQL persistence

Author: snazy

Diff for: LICENSE

@@ -304,6 +304,9 @@ This product includes code from Project Nessie.
 * tools/config-docs/generator/src/test/java/tests/smallrye/SomeEnum.java
 * tools/config-docs/generator/src/test/java/tests/smallrye/VeryNested.java
 
+Code underneath the components/persistence directory, especially pluggable object types, index related, cache,
+atomic commit logic and fundamental persistence implementations.
+
 Copyright: Copyright 2015-2025 Dremio Corporation
 Home page: https://projectnessie.org/
 License: https://www.apache.org/licenses/LICENSE-2.0

Diff for: bom/build.gradle.kts

@@ -43,6 +43,40 @@ dependencies {
     api(project(":polaris-idgen-impl"))
     api(project(":polaris-idgen-spi"))
 
+    api(project(":polaris-nodes-api"))
+    api(project(":polaris-nodes-impl"))
+    api(project(":polaris-nodes-spi"))
+    api(project(":polaris-nodes-store"))
+
+    api(project(":polaris-realms-api"))
+    api(project(":polaris-realms-impl"))
+    api(project(":polaris-realms-spi"))
+    api(project(":polaris-realms-store"))
+
+    api(project(":polaris-authz-api"))
+    api(project(":polaris-authz-impl"))
+    api(project(":polaris-authz-spi"))
+    api(project(":polaris-authz-store"))
+
+    api(project(":polaris-persistence-api"))
+    api(project(":polaris-persistence-impl"))
+    api(project(":polaris-persistence-benchmark"))
+    api(project(":polaris-persistence-bridge"))
+    api(project(":polaris-persistence-cache"))
+    api(project(":polaris-persistence-cdi-common"))
+    api(project(":polaris-persistence-cdi-quarkus"))
+    api(project(":polaris-persistence-cdi-weld"))
+    api(project(":polaris-persistence-commits"))
+    api(project(":polaris-persistence-correctness"))
+    api(project(":polaris-persistence-delegate"))
+    api(project(":polaris-persistence-index"))
+    api(project(":polaris-persistence-standalone"))
+    api(project(":polaris-persistence-testextension"))
+    api(project(":polaris-persistence-types"))
+
+    api(project(":polaris-persistence-inmemory"))
+    api(project(":polaris-persistence-mongodb"))
+
     api(project(":polaris-config-docs-annotations"))
     api(project(":polaris-config-docs-generator"))
 

Diff for: build.gradle.kts

@@ -72,6 +72,10 @@ tasks.named<RatTask>("rat").configure {
   // Manifest files do not allow comments
   excludes.add("tools/version/src/jarTest/resources/META-INF/FAKE_MANIFEST.MF")
 
+  excludes.add(
+    "components/persistence/index/src/testFixtures/resources/org/apache/polaris/persistence/indexes/words.gz"
+  )
+
   excludes.add("ide-name.txt")
   excludes.add("version.txt")
   excludes.add(".git")

Diff for: components/authz/README.md

@@ -0,0 +1,64 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# AuthZ framework with pluggable privileges
+
+Provides a framework and implementations pluggable privileges and privilege checks.
+
+## Privileges
+
+A privilege is globally identified by its name. Privileges can be inheritable (from its parents) or not. Multiple
+privileges can be grouped together to a _composite_ privilege (think: `ALL_DML` having `SELECT`, `INSERT`, `UPDATE` and
+`DELETE`) - a composite privilege matches, if all its individual privileges match. Multiple privileges can also be
+grouped to an _alternative_ privilege, which matches if any of its individual privileges matches.
+
+Available privileges are provided by one or more `PrivilegeProvider`s, which are discovered at runtime.
+Note: currently there is only one `ProvilegeProvider` that plugs in the Polaris privileges.
+
+## ACLs, ACL entries and ACL chains
+
+Each securable object can have its own ACL. ACLs consist of ACL entries, which define the _granted_ and _restricted_
+privileges by role name. The the number of roles is technically unbounded and the number of ACL entries can become
+quite large.
+
+This framework implements [separation of duties](https://en.wikipedia.org/wiki/Separation_of_duties) ("SoD"), which is a
+quite demanded functionality not just by large(r) user organizations. TL;DR _SoD_ allows "security administrators" to
+grant and revoke privileges to other users, but not leverage those privileges themselves.
+
+The _effective_ set of privileges for a specific operation performed by a specific caller needs to be computed against
+the target objects and their parents. _ACL chains_ are the vehicle to model this hierarchy and let the implementation
+compute the set of _effective_ privileges based on the individual ACLs and roles.
+
+Note: Privilege checks and _SoD_ are currently not performed via this framework.
+
+## Jackson support & Storage friendly representation
+
+The persistable types `Acl`, `AclEntry`, and `PrivilegeSet` can all be serialized using Jackson.
+
+As the number of ACL entries can become quite large, space efficient serialization is quite important. The
+implementation uses bit-set encoding when serializing `PrivilegeSet`s for persistence.
+
+## Code structure
+
+The code is structured into multiple modules. Consuming code should almost always pull in only the API module.
+
+* `polaris-authz-api` provides the necessary Java interfaces and immutable types.
+* `polaris-authz-impl` provides the storage agnostic implementation.
+* `polaris-authz-spi` provides the necessary interfaces to provide custom privileges and storage implementation.
+* `polaris-authz-store` provides the storage implementation based on `polaris-persistence-api`.

Diff for: components/authz/api/build.gradle.kts

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  alias(libs.plugins.jandex)
+  id("polaris-server")
+}
+
+description = "Polaris AuthZ API"
+
+dependencies {
+  implementation(libs.guava)
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+
+  compileOnly(libs.jakarta.annotation.api)
+  compileOnly(libs.jakarta.validation.api)
+  compileOnly(libs.jakarta.inject.api)
+  compileOnly(libs.jakarta.enterprise.cdi.api)
+
+  compileOnly(project(":polaris-immutables"))
+  annotationProcessor(project(":polaris-immutables", configuration = "processor"))
+}

Diff for: components/authz/api/src/main/java/org/apache/polaris/authz/api/Acl.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.authz.api;
+
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import jakarta.annotation.Nonnull;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+public interface Acl {
+
+  void entriesForRoleIds(
+      @Nonnull Set<String> roleIds, @Nonnull Consumer<AclEntry> aclEntryConsumer);
+
+  void forEach(@Nonnull BiConsumer<String, AclEntry> consumer);
+
+  interface AclBuilder {
+    @CanIgnoreReturnValue
+    AclBuilder from(@Nonnull Acl instance);
+
+    @CanIgnoreReturnValue
+    AclBuilder addEntry(@Nonnull String roleId, @Nonnull AclEntry entry);
+
+    @CanIgnoreReturnValue
+    AclBuilder removeEntry(@Nonnull String roleId);
+
+    /**
+     * Add, remove or update an {@linkplain AclEntry ACL entry} for a role.
+     *
+     * <p>The {@linkplain Consumer consumer} is called with an empty builder, if no ACL entry for
+     * the role exists, otherwise with a builder constructed from the existing entry. If the given
+     * {@linkplain Consumer consumer} removes all privileges from the ACL entry, the ACL entry will
+     * be removed.
+     */
+    @CanIgnoreReturnValue
+    AclBuilder modify(@Nonnull String roleId, @Nonnull Consumer<AclEntry.AclEntryBuilder> entry);
+
+    Acl build();
+  }
+}

Diff for: components/authz/api/src/main/java/org/apache/polaris/authz/api/AclChain.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.authz.api;
+
+import java.util.Optional;
+import org.apache.polaris.immutables.PolarisImmutable;
+import org.immutables.value.Value;
+
+/** Container for an {@linkplain Acl ACL} of an individual entity and a pointer to its parent. */
+@PolarisImmutable
+public interface AclChain {
+  @Value.Parameter(order = 1)
+  Acl acl();
+
+  @Value.Parameter(order = 2)
+  Optional<AclChain> parent();
+
+  static AclChain aclChain(Acl acl, Optional<AclChain> parent) {
+    return ImmutableAclChain.of(acl, parent);
+  }
+}

Diff for: components/authz/api/src/main/java/org/apache/polaris/authz/api/AclEntry.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.authz.api;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import jakarta.annotation.Nonnull;
+import java.util.Collection;
+import org.apache.polaris.immutables.PolarisImmutable;
+import org.immutables.value.Value;
+
+/**
+ * An {@link Acl ACL} entry holds the {@linkplain PrivilegeSet sets} of <em>granted</em> and
+ * <em>restricted</em> ("separation of duties") {@linkplain Privilege privileges}.
+ */
+@PolarisImmutable
+@JsonSerialize(as = ImmutableAclEntry.class)
+@JsonDeserialize(as = ImmutableAclEntry.class)
+public interface AclEntry {
+  @Value.Parameter(order = 1)
+  @Value.Default
+  // The 'CUSTOM/valueFilter' combination is there to only include non-empty privilege sets
+  @JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = PrivilegeSetJsonFilter.class)
+  default PrivilegeSet granted() {
+    return PrivilegeSet.emptyPrivilegeSet();
+  }
+
+  @Value.Parameter(order = 2)
+  @Value.Default
+  // The 'CUSTOM/valueFilter' combination is there to only include non-empty privilege sets
+  @JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = PrivilegeSetJsonFilter.class)
+  default PrivilegeSet restricted() {
+    return PrivilegeSet.emptyPrivilegeSet();
+  }
+
+  @Value.NonAttribute
+  @JsonIgnore
+  default boolean isEmpty() {
+    return granted().isEmpty() && restricted().isEmpty();
+  }
+
+  interface AclEntryBuilder {
+    @CanIgnoreReturnValue
+    AclEntryBuilder grant(@Nonnull Collection<? extends Privilege> privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder grant(@Nonnull Privilege... privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder grant(@Nonnull Privilege privilege);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder grant(@Nonnull PrivilegeSet privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder revoke(@Nonnull Collection<? extends Privilege> privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder revoke(@Nonnull Privilege... privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder revoke(@Nonnull Privilege privilege);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder revoke(@Nonnull PrivilegeSet privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder restrict(@Nonnull Collection<? extends Privilege> privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder restrict(@Nonnull Privilege... privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder restrict(@Nonnull Privilege privilege);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder restrict(@Nonnull PrivilegeSet privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder unrestrict(@Nonnull Collection<? extends Privilege> privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder unrestrict(@Nonnull Privilege... privileges);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder unrestrict(@Nonnull Privilege privilege);
+
+    @CanIgnoreReturnValue
+    AclEntryBuilder unrestrict(@Nonnull PrivilegeSet privileges);
+
+    AclEntry build();
+  }
+}