Integrated keystore

Author: Peter Alfonsi

server/src/main/java/org/opensearch/common/cache/tier/EhCacheDiskCachingTier.java

@@ -14,6 +14,7 @@
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.cache.tier.keystore.RBMIntKeyLookupStore;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
@@ -42,6 +43,7 @@
 import org.ehcache.event.EventType;
 import org.ehcache.expiry.ExpiryPolicy;
 import org.ehcache.impl.config.store.disk.OffHeapDiskStoreConfiguration;
+import org.opensearch.core.common.unit.ByteSizeValue;
 
 /**
  * @param <K> The key type of cache entries
@@ -92,6 +94,7 @@ public class EhCacheDiskCachingTier<K, V> implements DiskCachingTier<K, V> {
     // Defines how many segments the disk cache is separated into. Higher number achieves greater concurrency but
     // will hold that many file pointers.
     public final Setting<Integer> DISK_SEGMENTS;
+    private final RBMIntKeyLookupStore keystore;
 
     private final Serializer<K, byte[]> keySerializer;
     private final Serializer<V, byte[]> valueSerializer;
@@ -124,6 +127,11 @@ private EhCacheDiskCachingTier(Builder<K, V> builder) {
         close();
         cacheManager = buildCacheManager();
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
+
+        // IndicesRequestCache gets 1%, of which we allocate 5% to the keystore = 0.05%
+        // TODO: how do we change this automatically based on INDICES_CACHE_QUERY_SIZE setting?
+        Setting<ByteSizeValue> keystoreSizeSetting = Setting.memorySizeSetting(builder.settingPrefix + ".tiered.disk.keystore_size", "0.05%");
+        this.keystore = new RBMIntKeyLookupStore(keystoreSizeSetting.get(this.settings).getBytes());
     }
 
     private PersistentCacheManager buildCacheManager() {
@@ -193,12 +201,16 @@ private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder<
 
     @Override
     public V get(K key) {
-        return valueSerializer.deserialize(cache.get(key));
+        if (keystore.contains(key.hashCode())) { // Check in-memory store of key hashes to avoid unnecessary disk seek
+            return valueSerializer.deserialize(cache.get(key));
+        }
+        return null;
     }
 
     @Override
     public void put(K key, V value) {
         cache.put(key, valueSerializer.serialize(value));
+        keystore.add(key.hashCode());
     }
 
     @Override
@@ -211,6 +223,7 @@ public V computeIfAbsent(K key, TieredCacheLoader<K, V> loader) throws Exception
     public void invalidate(K key) {
         // There seems to be a thread leak issue while calling this and then closing cache.
         cache.remove(key);
+        keystore.remove(key.hashCode());
     }
 
     @Override
@@ -227,6 +240,7 @@ public void setRemovalListener(RemovalListener<K, V> removalListener) {
     @Override
     public void invalidateAll() {
         // Clear up files.
+        keystore.clear();
     }
 
     @Override

server/src/main/java/org/opensearch/common/cache/tier/keystore/KeyLookupStore.java

@@ -47,15 +47,15 @@ public interface KeyLookupStore<T> {
      * @return true if the value was added, false if it wasn't added because of a
      * collision or if it was already present.
      */
-    boolean add(T value) throws Exception;
+    boolean add(T value);
 
     /**
      * Checks if the transformation of the value is in the keystore.
      * @param value The value to check.
      * @return true if the value was found, false otherwise. Due to collisions, false positives are
      * possible, but there should be no false negatives unless forceRemove() is called.
      */
-    boolean contains(T value) throws Exception;
+    boolean contains(T value);
 
     /**
      * Returns the transformed version of the input value, that would be used to stored it in the keystore.
@@ -72,7 +72,7 @@ public interface KeyLookupStore<T> {
      * @param value The value to attempt to remove.
      * @return true if the value was removed, false if it wasn't.
      */
-    boolean remove(T value) throws Exception;
+    boolean remove(T value);
 
     /**
      * Returns the number of distinct values stored in the internal data structure.
@@ -123,10 +123,10 @@ public interface KeyLookupStore<T> {
      * Also resets all stats related to adding.
      * @param newValues The keys that should be in the reset structure.
      */
-    void regenerateStore(T[] newValues) throws Exception;
+    void regenerateStore(T[] newValues);
 
     /**
      * Deletes all keys and resets all stats related to adding.
      */
-    void clear() throws Exception;
+    void clear();
 }

server/src/main/java/org/opensearch/common/cache/tier/keystore/RBMIntKeyLookupStore.java

@@ -120,7 +120,7 @@ private void handleCollisions(int transformedValue) {
     }
 
     @Override
-    public boolean add(Integer value) throws Exception {
+    public boolean add(Integer value) {
         if (value == null) {
             return false;
         }
@@ -159,7 +159,7 @@ public boolean add(Integer value) throws Exception {
     }
 
     @Override
-    public boolean contains(Integer value) throws Exception {
+    public boolean contains(Integer value) {
         if (value == null) {
             return false;
         }
@@ -185,10 +185,9 @@ public Integer getInternalRepresentation(Integer value) {
      * may cause undefined behavior, including future false negatives!!
      * @param value The value to attempt to remove.
      * @return true if the value was removed, false otherwise
-     * @throws Exception
      */
     @Override
-    public boolean remove(Integer value) throws Exception {
+    public boolean remove(Integer value) {
         if (value == null) {
             return false;
         }
@@ -290,7 +289,7 @@ public boolean isFull() {
     }
 
     @Override
-    public void regenerateStore(Integer[] newValues) throws Exception {
+    public void regenerateStore(Integer[] newValues) {
         rbm.clear();
         collidedIntCounters = new HashMap<>();
         removalSets = new HashMap<>();
@@ -308,7 +307,7 @@ public void regenerateStore(Integer[] newValues) throws Exception {
     }
 
     @Override
-    public void clear() throws Exception {
+    public void clear() {
         regenerateStore(new Integer[] {});
     }