Add dumper/loader for basic yaml (lists of lists) (#11)

Author: rly

poetry.lock

@@ -38,6 +38,11 @@ docs = [
     "myst-parser"
     ]
 
+[tool.poetry.group.dev.dependencies]
+black = "^24.1.1"
+pytest = "^7.1.2"
+mypy = "^1.8.0"
+
 [tool.poetry-dynamic-versioning]
 enable = true
 vcs = "git"

pyproject.toml

@@ -1,12 +1,14 @@
 """Dumper classes for linkml-arrays."""
 
 from .hdf5_dumper import Hdf5Dumper
+from .yaml_dumper import YamlDumper
 from .yaml_hdf5_dumper import YamlHdf5Dumper
 from .yaml_numpy_dumper import YamlNumpyDumper
 from .zarr_directory_store_dumper import ZarrDirectoryStoreDumper
 
 __all__ = [
     "Hdf5Dumper",
+    "YamlDumper",
     "YamlHdf5Dumper",
     "YamlNumpyDumper",
     "ZarrDirectoryStoreDumper",

src/linkml_arrays/dumpers/__init__.py

@@ -0,0 +1,58 @@
+"""Class for dumpling a LinkML model to a YAML file."""
+
+from typing import Union
+
+import yaml
+from linkml_runtime import SchemaView
+from linkml_runtime.dumpers.dumper_root import Dumper
+from linkml_runtime.utils.yamlutils import YAMLRoot
+from pydantic import BaseModel
+
+
+def _iterate_element(
+    element: Union[YAMLRoot, BaseModel], schemaview: SchemaView, parent_identifier=None
+):
+    """Recursively iterate through the elements of a LinkML model and save them.
+
+    Returns a dictionary with the same structure as the input element, but with the slots
+    that implement "linkml:elements" (arrays) are written as lists or lists of lists.
+
+    Raises:
+        ValueError: If the class requires an identifier and it is not provided.
+    """
+    # get the type of the element
+    element_type = type(element).__name__
+
+    # ask schemaview whether it has a class by this name
+    found_class = schemaview.get_class(element_type)
+
+    id_slot = schemaview.get_identifier_slot(found_class.name)
+    if id_slot is not None:
+        id_value = getattr(element, id_slot.name)
+    else:
+        id_value = None
+
+    ret_dict = dict()
+    for k, v in vars(element).items():
+        found_slot = schemaview.induced_slot(k, element_type)
+        if "linkml:elements" in found_slot.implements:
+            if id_slot is None and parent_identifier is None:
+                raise ValueError("The class requires an identifier.")
+            ret_dict[k] = v.tolist()
+        else:
+            if isinstance(v, BaseModel):
+                v2 = _iterate_element(v, schemaview, id_value)
+                ret_dict[k] = v2
+            else:
+                ret_dict[k] = v
+    return ret_dict
+
+
+class YamlDumper(Dumper):
+    """Dumper class for LinkML models to YAML files."""
+
+    def dumps(self, element: Union[YAMLRoot, BaseModel], schemaview: SchemaView, **kwargs) -> str:
+        """Return element formatted as a YAML string."""
+        input = _iterate_element(element, schemaview)
+
+        return yaml.dump(input)

src/linkml_arrays/dumpers/yaml_dumper.py

@@ -2,12 +2,14 @@
 
 from .hdf5_loader import Hdf5Loader
 from .yaml_hdf5_loader import YamlHdf5Loader
+from .yaml_loader import YamlLoader
 from .yaml_numpy_loader import YamlNumpyLoader
 from .zarr_directory_store_loader import ZarrDirectoryStoreLoader
 
 __all__ = [
     "Hdf5Loader",
     "YamlHdf5Loader",
+    "YamlLoader",
     "YamlNumpyLoader",
     "ZarrDirectoryStoreLoader",
 ]

src/linkml_arrays/loaders/__init__.py

@@ -0,0 +1,60 @@
+"""Class for loading a LinkML model from a YAML file."""
+
+from typing import Type, Union
+
+import numpy as np
+import yaml
+from linkml_runtime import SchemaView
+from linkml_runtime.linkml_model import ClassDefinition
+from linkml_runtime.loaders.loader_root import Loader
+from linkml_runtime.utils.yamlutils import YAMLRoot
+from pydantic import BaseModel
+
+
+def _iterate_element(
+    input_dict: dict, element_type: ClassDefinition, schemaview: SchemaView
+) -> dict:
+    """Recursively iterate through the elements of a LinkML model and load them into a dict.
+
+    Datasets are loaded into NumPy arrays.
+    """
+    ret_dict = dict()
+    for k, v in input_dict.items():
+        found_slot = schemaview.induced_slot(k, element_type.name)
+        if "linkml:elements" in found_slot.implements:
+            v = np.asarray(v)
+        elif isinstance(v, dict):
+            found_slot_range = schemaview.get_class(found_slot.range)
+            v = _iterate_element(v, found_slot_range, schemaview)
+        # else: do not transform v
+        ret_dict[k] = v
+
+    return ret_dict
+
+
+class YamlLoader(Loader):
+    """Class for loading a LinkML model from a YAML file."""
+
+    def load_any(self, source: str, **kwargs):
+        """Create an instance of the target class from a YAML file."""
+        return self.load(source, **kwargs)
+
+    def loads(self, source: str, **kwargs):
+        """Create an instance of the target class from a YAML file."""
+        return self.load(source, **kwargs)
+
+    def load(
+        self,
+        source: str,
+        target_class: Type[Union[YAMLRoot, BaseModel]],
+        schemaview: SchemaView,
+        **kwargs,
+    ):
+        """Create an instance of the target class from a YAML file."""
+        input_dict = yaml.safe_load(source)
+
+        element_type = schemaview.get_class(target_class.__name__)
+        element = _iterate_element(input_dict, element_type, schemaview)
+        obj = target_class(**element)
+
+        return obj

src/linkml_arrays/loaders/yaml_loader.py

@@ -0,0 +1,23 @@
+latitude_in_deg:
+  values:
+  - 1
+  - 2
+longitude_in_deg:
+  values:
+  - 4
+  - 5
+name: my_temperature
+temperatures_in_K:
+  values:
+  - - - 0
+      - 1
+    - - 2
+      - 3
+  - - - 4
+      - 5
+    - - 6
+      - 7
+time_in_d:
+  values:
+  - 7
+  - 8

tests/input/temperature_dataset_yaml.yaml

@@ -9,6 +9,7 @@
 
 from linkml_arrays.dumpers import (
     Hdf5Dumper,
+    YamlDumper,
     YamlHdf5Dumper,
     YamlNumpyDumper,
     ZarrDirectoryStoreDumper,
@@ -22,6 +23,55 @@
 )
 
 
+class YamlDumpersTestCase(unittest.TestCase):
+    """Test dumping of pydantic-style classes from LinkML schemas into YAML files."""
+
+    def test_dump_pydantic_arrays(self):
+        """Test dumping pydantic classes with numpy arrays to a YAML file."""
+        latitude_in_deg = LatitudeSeries(values=np.array([1, 2]))
+        longitude_in_deg = LongitudeSeries(values=np.array([4, 5]))
+        time_in_d = DaySeries(values=np.array([7, 8]))
+        temperatures_in_K = TemperatureMatrix(
+            values=np.arange(8).reshape((2, 2, 2)),
+        )
+        temperature = TemperatureDataset(
+            name="my_temperature",
+            latitude_in_deg=latitude_in_deg,
+            longitude_in_deg=longitude_in_deg,
+            time_in_d=time_in_d,
+            temperatures_in_K=temperatures_in_K,
+        )
+
+        schemaview = SchemaView(Path(__file__) / "../../input/temperature_dataset.yaml")
+        ret = YamlDumper().dumps(temperature, schemaview=schemaview)
+
+        expected = """latitude_in_deg:
+  values:
+  - 1
+  - 2
+longitude_in_deg:
+  values:
+  - 4
+  - 5
+name: my_temperature
+temperatures_in_K:
+  values:
+  - - - 0
+      - 1
+    - - 2
+      - 3
+  - - - 4
+      - 5
+    - - 6
+      - 7
+time_in_d:
+  values:
+  - 7
+  - 8
+"""
+        assert ret == expected
+
+
 class YamlNumpyDumpersTestCase(unittest.TestCase):
     """Test dumping of pydantic-style classes from LinkML schemas into YAML + NumPy files."""
 

tests/test_dumpers/test_dumpers.py

@@ -10,6 +10,7 @@
 from linkml_arrays.loaders import (
     Hdf5Loader,
     YamlHdf5Loader,
+    YamlLoader,
     YamlNumpyLoader,
     ZarrDirectoryStoreLoader,
 )
@@ -22,6 +23,33 @@
 )
 
 
+class YamlLoadersTestCase(unittest.TestCase):
+    """Test loading of pydantic-style classes from YAML arrays."""
+
+    def test_load_pydantic_arrays(self):
+        """Test loading of pydantic-style classes from YAML arrays."""
+        read_yaml = hbread(
+            "temperature_dataset_yaml.yaml", base_path=str(Path(__file__) / "../../input")
+        )
+        schemaview = SchemaView(Path(__file__) / "../../input/temperature_dataset.yaml")
+        ret = YamlLoader().loads(read_yaml, target_class=TemperatureDataset, schemaview=schemaview)
+
+        assert isinstance(ret, TemperatureDataset)
+        assert ret.name == "my_temperature"
+
+        assert isinstance(ret.latitude_in_deg, LatitudeSeries)
+        np.testing.assert_array_equal(ret.latitude_in_deg.values, np.array([1, 2]))
+
+        assert isinstance(ret.longitude_in_deg, LongitudeSeries)
+        np.testing.assert_array_equal(ret.longitude_in_deg.values, np.array([4, 5]))
+
+        assert isinstance(ret.time_in_d, DaySeries)
+        np.testing.assert_array_equal(ret.time_in_d.values, np.array([7, 8]))
+
+        assert isinstance(ret.temperatures_in_K, TemperatureMatrix)
+        np.testing.assert_array_equal(ret.temperatures_in_K.values, np.arange(8).reshape((2, 2, 2)))
+
+
 class YamlNumpyLoadersTestCase(unittest.TestCase):
     """Test loading of pydantic-style classes from YAML + Numpy arrays."""