Bug 38279239 - [37442204->14.1.1.0.23] Snapshot validation passes for snapshots can't be recovered (missing partition folders aren't detected) (14.1.1.0 cl 118395 --> 14.1.1.0 CE)

[git-p4: depot-paths = "//dev/coherence-ce/release/coherence-ce-v14.1.1.0/": change = 118449]

Author: fryp

prj/coherence-core/src/main/java/com/tangosol/io/FileHelper.java

@@ -1,8 +1,8 @@
 /*
- * Copyright (c) 2000, 2020, Oracle and/or its affiliates.
+ * Copyright (c) 2000, 2025, Oracle and/or its affiliates.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at
- * http://oss.oracle.com/licenses/upl.
+ * https://oss.oracle.com/licenses/upl.
  */
 package com.tangosol.io;
 
@@ -844,6 +844,22 @@ public static String removeExtension(final String filename)
             }
         }
 
+    /**
+     * Return true if the specified directory is empty.
+     *
+     * @param  fileDir the directory to check
+     *
+     * @return true if the directory is empty
+     *
+     * @since 24.09
+     */
+    public static boolean isEmpty(File fileDir)
+        {
+        File[] afile = fileDir.listFiles();
+
+        return fileDir.isDirectory() && (afile == null || afile.length == 0);
+        }
+
     // ----- static helper classes ------------------------------------------
 
     /**

prj/coherence-core/src/main/java/com/tangosol/persistence/AbstractPersistenceManager.java

@@ -37,6 +37,7 @@
 import com.tangosol.util.Base;
 import com.tangosol.util.ClassHelper;
 import com.tangosol.util.NullImplementation;
+import com.tangosol.util.SimpleMapEntry;
 
 import java.io.DataInputStream;
 import java.io.DataOutput;
@@ -51,7 +52,10 @@
 import java.nio.channels.FileLock;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Deque;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -567,24 +571,37 @@ public PersistenceTools getPersistenceTools()
             throw new IllegalArgumentException("snapshot must have at least one GUID");
             }
 
-        String sGUID = asGUIDs[0];
-        int    nVersion;
-        int    cPartitions;
+        int      nVersion;
+        int      cPartitions;
+        FileLock fileLock    = null;
+        String   sGUID       = null;
 
         PersistentStore<ReadBuffer> store = null;
 
         try
             {
+            Map.Entry<String, FileLock> entry = pickStore(asGUIDs);
+            fileLock = entry.getValue();
+            sGUID    = entry.getKey();
+
             store = open(sGUID, null);
             cPartitions = CachePersistenceHelper.getPartitionCount(store);
             nVersion    = CachePersistenceHelper.getPersistenceVersion(store);
             }
+        catch (Exception e)
+            {
+            throw Base.ensureRuntimeException(e);
+            }
         finally
             {
             if (store != null)
                 {
                 close(sGUID);
                 }
+            if (fileLock != null)
+                {
+                FileHelper.unlockFile(fileLock);
+                }
             }
 
         OfflinePersistenceInfo info = new OfflinePersistenceInfo(cPartitions, getStorageFormat(),
@@ -593,6 +610,23 @@ public PersistenceTools getPersistenceTools()
         return instantiatePersistenceTools(info);
         }
 
+    private Map.Entry<String, FileLock> pickStore(String[] stores)
+        {
+        Deque<String> candidates = new LinkedList<>(Arrays.asList(stores));
+        String guid;
+        while((guid = candidates.poll()) != null)
+            {
+            File lockDir      = getLockDirectory();
+            File lockFile     = new File(lockDir, guid + ".store.lck");
+            FileLock fileLock = FileHelper.lockFile(lockFile);
+            if (fileLock != null)
+                {
+                return new SimpleMapEntry<>(guid, fileLock);
+                }
+            }
+        throw new RuntimeException("all stores are locked");
+        }
+
     /**
      * {@inheritDoc}
      */

prj/coherence-core/src/main/java/com/tangosol/persistence/CachePersistenceHelper.java

@@ -64,6 +64,9 @@
 import java.io.Reader;
 import java.io.Writer;
 
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -1150,6 +1153,22 @@ public static void resumeService(final Cluster cluster, final String sServiceNam
      * @throws PersistenceException if any errors
      */
     public static PersistenceTools getSnapshotPersistenceTools(File dirSnapshot)
+        {
+        return getSnapshotPersistenceTools(dirSnapshot, false);
+        }
+
+    /**
+     * Return an implementation specific instance of {@link PersistenceTools} for
+     * the given local snapshot directory.
+     *
+     * @param dirSnapshot  the snapshot directory to get tools for
+     * @param validation   whether to enable snapshot validation
+     *
+     * @return an implementation specific instance of PersistenceTools
+     *
+     * @throws PersistenceException if any errors
+     */
+    public static PersistenceTools getSnapshotPersistenceTools(File dirSnapshot, boolean validation)
         {
         PersistenceTools tools;
 
@@ -1192,7 +1211,7 @@ public static PersistenceTools getSnapshotPersistenceTools(File dirSnapshot)
 
             if ("BDB".equals(sPersistenceType))
                 {
-                tools = new BerkeleyDBManager(dirSnapshot, null, null).getPersistenceTools();
+                tools = new BerkeleyDBManager(dirSnapshot, null, null, validation).getPersistenceTools();
                 }
             else
                 {
@@ -1544,7 +1563,26 @@ public static String[] getFailedSnapshots(PersistenceEnvironment env)
             File dirSnapshot = new File(dirSnapshots, sName);
             try
                 {
-                getSnapshotPersistenceTools(dirSnapshot).validate();
+                PersistenceTools pt = getSnapshotPersistenceTools(dirSnapshot, true);
+                try 
+                    {
+                    // Using reflection because the PersistenceTools interface wasn't updated,
+                    // and BerkeleyDBManager returns an anonymous class as the PersistenceTools
+                    // implementation, so casting isn't possible.
+                    Method validate = pt.getClass().getMethod("validateWithPartitions");
+                    validate.setAccessible(true);
+                    String[] checkedPartitions = (String[]) validate.invoke(pt);
+                    String validationResult = String.format("%s~~~[%s]", sName, String.join(",", checkedPartitions));
+                    asFailedSnapshots.add(validationResult);
+                    }
+                catch (NoSuchMethodException e)
+                    {
+                    getSnapshotPersistenceTools(dirSnapshot, true).validate();
+                    }
+                catch (InvocationTargetException | IllegalAccessException e)
+                    {
+                    throw Base.ensureRuntimeException(e);
+                    }
                 }
             catch (RuntimeException e)
                 {