Clean up test files: dedicated VASP directories, xyz, mcif, cssr, exciting, wannier90 (materialsproject#3681)

* relocate fixture dir: absorption

* relocate dir: absorption

* relocate `BTO_221_99_polarization` dir

* remove unused `chgden`

* remove unused `defect`

* relocate `fe_monomer`

* relocate

* relocate `fake_potcars`

* Reapply "relocate fake_potcars"

This reverts commit 61d6cd3.

* relocate `grid_data_files`

* relocate `kpoints_opt`

* relocate `neb_analysis`

* fix path typo

* remove unused `path_finder`

* relocate `relaxation`

* relocate `reproduce_eps`

* fix same typo again

* fix neb file path

* relocate `scan_relaxation`

* relocate `static_silicon`

* remove unused `Zr_Vasprun`

* relocate `thermal_displacement_matrices`

* remove unused `ion_placer` from qchem

* relocate VASP output WSWQ

* remove unused file `sio2.pw.in`

* relocate fiesta file

* relocate some `xyz` files

* relocate `exciting` files

* relocate cssr files

* relocate `wannier90`files

* relocate multiple mcif files

* try gzip vasprun.xml in vasp fixtures

* try removing skip `matgl`

* Revert "try removing skip `matgl`"

This reverts commit 27a4eeb.

* decompress vasprun for optics

* relocate a mcif file

* relocate gulp and remove moved mcif

* create dir for `borg`

* create dedicated `borg` test dir

* removed unnecessary self.

* fix path and use stronger assert

* revert vasprun to "vasp run"

* add type hints

---------

Co-authored-by: Janosh Riebesell <[email protected]>

Author: DanielYang59

dev_scripts/potcar_scrambler.py

@@ -12,6 +12,7 @@
 from pymatgen.core import SETTINGS
 from pymatgen.io.vasp import Potcar, PotcarSingle
 from pymatgen.io.vasp.sets import _load_yaml_config
+from pymatgen.util.testing import VASP_IN_DIR
 
 
 class PotcarScrambler:
@@ -180,7 +181,7 @@ def potcar_cleanser():
     and freely shared by VASP)
     """
 
-    search_dir = "../tests/files/fake_potcars/real_potcars/"
+    search_dir = f"{VASP_IN_DIR}/fake_potcars/real_potcars/"
     rebase_dir = search_dir.replace("real", "fake")
     potcars_to_cleanse = glob(f"{search_dir}/**/POTCAR*", recursive=True)
 

pymatgen/apps/borg/queen.py

@@ -9,10 +9,14 @@
 import logging
 import os
 from multiprocessing import Manager, Pool
+from typing import TYPE_CHECKING
 
 from monty.io import zopen
 from monty.json import MontyDecoder, MontyEncoder
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 logger = logging.getLogger("BorgQueen")
 
 
@@ -60,33 +64,33 @@ def parallel_assimilate(self, rootpath):
         status["count"] = 0
         status["total"] = len(valid_paths)
         logger.info(f"{len(valid_paths)} valid paths found.")
-        with Pool(self._num_drones) as p:
-            p.map(
+        with Pool(self._num_drones) as pool:
+            pool.map(
                 order_assimilation,
                 ((path, self._drone, data, status) for path in valid_paths),
             )
-            for d in data:
-                self._data.append(json.loads(d, cls=MontyDecoder))
+            for string in data:
+                self._data.append(json.loads(string, cls=MontyDecoder))
 
-    def serial_assimilate(self, rootpath):
+    def serial_assimilate(self, root: str | Path) -> None:
         """Assimilate the entire subdirectory structure in rootpath serially."""
         valid_paths = []
-        for parent, subdirs, files in os.walk(rootpath):
+        for parent, subdirs, files in os.walk(root):
             valid_paths.extend(self._drone.get_valid_paths((parent, subdirs, files)))
-        data = []
+        data: list[str] = []
         total = len(valid_paths)
         for idx, path in enumerate(valid_paths, 1):
             new_data = self._drone.assimilate(path)
             self._data.append(new_data)
-            logger.info(f"{idx}/{total} ({idx / total:.2%}) done")
+            logger.info(f"{idx}/{total} ({idx / total:.1%}) done")
         for json_str in data:
             self._data.append(json.loads(json_str, cls=MontyDecoder))
 
     def get_data(self):
         """Returns an list of assimilated objects."""
         return self._data
 
-    def save_data(self, filename):
+    def save_data(self, filename: str | Path) -> None:
         """Save the assimilated data to a file.
 
         Args:

pymatgen/util/testing/__init__.py

@@ -31,7 +31,7 @@
 # fake POTCARs have original header information, meaning properties like number of electrons,
 # nuclear charge, core radii, etc. are unchanged (important for testing) while values of the and
 # pseudopotential kinetic energy corrections are scrambled to avoid VASP copyright infringement
-FAKE_POTCAR_DIR = TEST_FILES_DIR / "fake_potcars"
+FAKE_POTCAR_DIR = f"{VASP_IN_DIR}/fake_potcars"
 
 
 class PymatgenTest(unittest.TestCase):

tests/analysis/ferroelectricity/test_polarization.py

@@ -15,7 +15,7 @@
 from pymatgen.io.vasp.outputs import Outcar
 from pymatgen.util.testing import TEST_FILES_DIR, PymatgenTest
 
-TEST_DIR = f"{TEST_FILES_DIR}/BTO_221_99_polarization"
+TEST_DIR = f"{TEST_FILES_DIR}/vasp/fixtures/BTO_221_99_polarization"
 bto_folders = ["nonpolar_polarization"]
 bto_folders += [f"interpolation_{i}_polarization" for i in range(1, 9)][::-1]
 bto_folders += ["polar_polarization"]

tests/analysis/magnetism/test_analyzer.py

@@ -30,13 +30,13 @@ def setUp(self):
 
         self.Fe3O4 = Structure.from_file(f"{TEST_FILES_DIR}/Fe3O4.cif", primitive=True)
 
-        self.GdB4 = Structure.from_file(f"{TEST_FILES_DIR}/magnetic.ncl.example.GdB4.mcif", primitive=True)
+        self.GdB4 = Structure.from_file(f"{TEST_FILES_DIR}/mcif/magnetic.ncl.example.GdB4.mcif", primitive=True)
 
-        self.NiO_expt = Structure.from_file(f"{TEST_FILES_DIR}/magnetic.example.NiO.mcif", primitive=True)
+        self.NiO_expt = Structure.from_file(f"{TEST_FILES_DIR}/mcif/magnetic.example.NiO.mcif", primitive=True)
 
         # CuO.mcif sourced from https://www.cryst.ehu.es/magndata/index.php?index=1.62
         # doi: 10.1088/0022-3719/21/15/023
-        self.CuO_expt = Structure.from_file(f"{TEST_FILES_DIR}/magnetic.example.CuO.mcif.gz", primitive=True)
+        self.CuO_expt = Structure.from_file(f"{TEST_FILES_DIR}/mcif/magnetic.example.CuO.mcif.gz", primitive=True)
 
         lattice = Lattice.cubic(4.17)
         species = ["Ni", "O"]

tests/analysis/test_local_env.py

@@ -287,8 +287,8 @@ def test_get_nn(self):
 class TestOpenBabelNN(PymatgenTest):
     def setUp(self):
         pytest.importorskip("openbabel")
-        self.benzene = Molecule.from_file(f"{TEST_FILES_DIR}/benzene.xyz")
-        self.acetylene = Molecule.from_file(f"{TEST_FILES_DIR}/acetylene.xyz")
+        self.benzene = Molecule.from_file(f"{TEST_FILES_DIR}/xyz/benzene.xyz")
+        self.acetylene = Molecule.from_file(f"{TEST_FILES_DIR}/xyz/acetylene.xyz")
 
     def test_nn_orders(self):
         strategy = OpenBabelNN()
@@ -318,8 +318,8 @@ def test_nn_length(self):
 
 class TestCovalentBondNN(PymatgenTest):
     def setUp(self):
-        self.benzene = Molecule.from_file(f"{TEST_FILES_DIR}/benzene.xyz")
-        self.acetylene = Molecule.from_file(f"{TEST_FILES_DIR}/acetylene.xyz")
+        self.benzene = Molecule.from_file(f"{TEST_FILES_DIR}/xyz/benzene.xyz")
+        self.acetylene = Molecule.from_file(f"{TEST_FILES_DIR}/xyz/acetylene.xyz")
 
     def test_nn_orders(self):
         strategy = CovalentBondNN()

tests/analysis/test_transition_state.py

@@ -20,7 +20,7 @@
 __date__ = "2/5/16"
 
 
-TEST_DIR = f"{TEST_FILES_DIR}/neb_analysis"
+TEST_DIR = f"{TEST_FILES_DIR}/vasp/fixtures/neb_analysis"
 
 
 class TestNEBAnalysis(PymatgenTest):

tests/apps/borg/test_hive.py

@@ -3,7 +3,6 @@
 import os
 import unittest
 
-from monty.tempfile import ScratchDir
 from pytest import approx
 
 from pymatgen.apps.borg.hive import (
@@ -26,25 +25,21 @@ def test_get_valid_paths(self):
                 assert len(self.drone.get_valid_paths(path)) > 0
 
     def test_assimilate(self):
-        """Test assimilate data from "vasprun.xe.xml.gz" file."""
-
-        with ScratchDir("."):
-            # Need to rename the test file to "vasprun.xml.xe.gz" as
-            # hive is looking for pattern "vasprun.xml*"
-            os.symlink(f"{VASP_OUT_DIR}/vasprun.xe.xml.gz", "vasprun.xml.xe.gz")
-            entry = self.drone.assimilate(".")
-
-            for param in ("hubbards", "is_hubbard", "potcar_spec", "run_type"):
-                assert param in entry.parameters
-            assert entry.data["efermi"] == approx(-6.62148548)
-            assert entry.reduced_formula == "Xe"
-            assert entry.energy == approx(0.5559329)
-
-            entry = self.structure_drone.assimilate(".")
-            assert entry.reduced_formula == "Xe"
-            assert entry.energy == approx(0.5559329)
-            assert isinstance(entry, ComputedStructureEntry)
-            assert entry.structure is not None
+        """Test assimilate data from "vasprun.xml.xe.gz" file."""
+
+        entry = self.drone.assimilate(f"{TEST_FILES_DIR}/app_borg/test_dir")
+
+        for param in ("hubbards", "is_hubbard", "potcar_spec", "run_type"):
+            assert param in entry.parameters
+        assert entry.data["efermi"] == approx(-6.62148548)
+        assert entry.reduced_formula == "Xe"
+        assert entry.energy == approx(0.5559329)
+
+        entry = self.structure_drone.assimilate(f"{TEST_FILES_DIR}/app_borg/test_dir")
+        assert entry.reduced_formula == "Xe"
+        assert entry.energy == approx(0.5559329)
+        assert isinstance(entry, ComputedStructureEntry)
+        assert entry.structure is not None
 
     def test_as_from_dict(self):
         dct = self.structure_drone.as_dict()
@@ -81,16 +76,16 @@ def test_get_valid_paths(self):
     def test_assimilate(self):
         test_file = f"{TEST_FILES_DIR}/molecules/methane.log"
         entry = self.drone.assimilate(test_file)
-        for p in [
+        for param in [
             "functional",
             "basis_set",
             "charge",
             "spin_multiplicity",
             "route_parameters",
         ]:
-            assert p in entry.parameters
-        for p in ["corrections"]:
-            assert p in entry.data
+            assert param in entry.parameters
+        for param in ["corrections"]:
+            assert param in entry.data
 
         assert entry.reduced_formula == "H4C"
         assert entry.energy == approx(-39.9768775602)
@@ -99,8 +94,8 @@ def test_assimilate(self):
         assert entry.energy == approx(-39.9768775602)
         assert isinstance(entry, ComputedStructureEntry)
         assert entry.structure is not None
-        for p in ["properly_terminated", "stationary_type"]:
-            assert p in entry.data
+        for param in ["properly_terminated", "stationary_type"]:
+            assert param in entry.data
 
     def test_as_from_dict(self):
         dct = self.structure_drone.as_dict()

tests/apps/borg/test_queen.py

@@ -1,5 +1,3 @@
-"""Created on Mar 18, 2012."""
-
 from __future__ import annotations
 
 import unittest
@@ -12,16 +10,20 @@
 __copyright__ = "Copyright 2012, The Materials Project"
 __date__ = "Mar 18, 2012"
 
+TEST_DIR = f"{TEST_FILES_DIR}/app_borg"
+
 
 class TestBorgQueen(unittest.TestCase):
     def test_get_data(self):
+        """Test get data from vasprun.xml.xe.gz file."""
         drone = VaspToComputedEntryDrone()
-        self.queen = BorgQueen(drone, TEST_FILES_DIR, 1)
-        data = self.queen.get_data()
-        assert len(data) == 16
+        queen = BorgQueen(drone, TEST_DIR, 1)
+        data = queen.get_data()
+        assert len(data) == 1
+        assert data[0].energy == 0.5559329
 
     def test_load_data(self):
         drone = VaspToComputedEntryDrone()
         queen = BorgQueen(drone)
-        queen.load_data(f"{TEST_FILES_DIR}/assimilated.json")
+        queen.load_data(f"{TEST_DIR}/assimilated.json")
         assert len(queen.get_data()) == 1

tests/command_line/test_gulp_caller.py

@@ -245,7 +245,7 @@ def test_get_energy(self):
     def test_get_relaxed_structure(self):
         # Output string obtained from running GULP on a terminal
 
-        with open(f"{TEST_FILES_DIR}/example21.gout") as file:
+        with open(f"{TEST_FILES_DIR}/gulp/example21.gout") as file:
             out_str = file.read()
         struct = self.gio.get_relaxed_structure(out_str)
         assert isinstance(struct, Structure)

tests/core/test_structure.py

@@ -874,12 +874,12 @@ def test_to_from_file_and_string(self):
 
         # test CIF file with unicode error
         # https://github.com/materialsproject/pymatgen/issues/2947
-        struct = Structure.from_file(f"{TEST_FILES_DIR}/bad-unicode-gh-2947.mcif")
+        struct = Structure.from_file(f"{TEST_FILES_DIR}/mcif/bad-unicode-gh-2947.mcif")
         assert struct.formula == "Ni32 O32"
 
         # make sure CIfParser.parse_structures() and Structure.from_file() are consistent
         # i.e. uses same merge_tol for site merging, same primitive=False, etc.
-        assert struct == CifParser(f"{TEST_FILES_DIR}/bad-unicode-gh-2947.mcif").parse_structures()[0]
+        assert struct == CifParser(f"{TEST_FILES_DIR}/mcif/bad-unicode-gh-2947.mcif").parse_structures()[0]
 
         # https://github.com/materialsproject/pymatgen/issues/3551
         json_path = Path("test-with-path.json")
@@ -1559,7 +1559,7 @@ def test_charge(self):
         assert super_cell.charge == 25, "Set charge not properly modifying _charge"
 
     def test_vesta_lattice_matrix(self):
-        silica_zeolite = Molecule.from_file(f"{TEST_FILES_DIR}/CON_vesta.xyz")
+        silica_zeolite = Molecule.from_file(f"{TEST_FILES_DIR}/xyz/CON_vesta.xyz")
 
         s_vesta = Structure(
             lattice=Lattice.from_parameters(22.6840, 13.3730, 12.5530, 90, 69.479, 90, vesta=True),

tests/electronic_structure/test_plotter.py

@@ -249,7 +249,7 @@ def test_methods(self):
 
 class TestPlotBZ(unittest.TestCase):
     def setUp(self):
-        self.rec_latt = Structure.from_file(f"{TEST_FILES_DIR}/Si.cssr").lattice.reciprocal_lattice
+        self.rec_latt = Structure.from_file(f"{TEST_FILES_DIR}/cssr/Si.cssr").lattice.reciprocal_lattice
         self.kpath = [[[0.0, 0.0, 0.0], [0.5, 0.0, 0.5], [0.5, 0.25, 0.75], [0.375, 0.375, 0.75]]]
         self.labels = {
             "\\Gamma": [0.0, 0.0, 0.0],