Implement measurement field conversion from Vision

src/power_grid_model_io/converters/tabular_converter.py

@@ -117,7 +117,7 @@ def _parse_data(self, data: TabularData, data_type: str, extra_info: Optional[Ex
 
         # For each table in the mapping
         for table in self._mapping.tables():
-            if table not in data or len(data[table]) == 0:
+            if table not in data or len(data[table][table_mask]) == 0:
                 continue  # pragma: no cover (bug in python 3.9)
             for component, attributes in self._mapping.instances(table=table):
                 component_data = self._convert_table_to_component(
@@ -178,7 +178,14 @@ def _convert_table_to_component(
         if table not in data:
             return None
 
-        n_records = len(data[table])
+        table_mask = np.full(len(data[table]), True)
+        if "filter" in attributes:
+            table_mask = self._parse_filters(data, table)
+
+        n_records = np.sum(table_mask)
+
+        if n_records == 0:
+            return None
 
         try:
             pgm_data = initialize_array(data_type=data_type, component_type=component, shape=n_records)
@@ -200,6 +207,7 @@ def _convert_table_to_component(
                 attr=attr,
                 col_def=col_def,
                 extra_info=extra_info,
+                table_mask=table_mask
             )
 
         return pgm_data
@@ -214,6 +222,7 @@ def _convert_col_def_to_attribute(
         attr: str,
         col_def: Any,
         extra_info: Optional[ExtraInfo],
+        table_mask: np.ndarray
     ):
         """This function updates one of the attributes of pgm_data, based on the corresponding table/column in a tabular
         dataset
@@ -242,33 +251,41 @@ def _convert_col_def_to_attribute(
         """
         # To avoid mistakes, the attributes in the mapping should exist. There is one extra attribute called
         # 'extra' in which extra information can be captured.
-        if attr not in pgm_data.dtype.names and attr != "extra":
+        if attr not in pgm_data.dtype.names and attr not in ["extra", "filter"]:
             attrs = ", ".join(pgm_data.dtype.names)
             raise KeyError(f"Could not find attribute '{attr}' for '{component}s'. (choose from: {attrs})")
 
         if attr == "extra":
             # Extra info must be linked to the object IDs, therefore the uuids should be known before extra info can
             # be parsed. Before this for loop, it is checked that "id" exists and it is placed at the front.
             self._handle_extra_info(
-                data=data, table=table, col_def=col_def, uuids=pgm_data["id"], extra_info=extra_info
+                data=data, table=table, col_def=col_def, uuids=pgm_data["id"], extra_info=extra_info, table_mask=table_mask
             )
             # Extra info should not be added to the numpy arrays, so let's continue to the next attribute
             return
 
-        attr_data = self._parse_col_def(data=data, table=table, col_def=col_def, extra_info=extra_info)
+        attr_data = self._parse_col_def(data=data, table=table, col_def=col_def, extra_info=extra_info, table_mask=table_mask)
 
         if len(attr_data.columns) != 1:
             raise ValueError(f"DataFrame for {component}.{attr} should contain a single column ({attr_data.columns})")
 
         pgm_data[attr] = attr_data.iloc[:, 0]
 
+    def _parse_filters(self) -> pd.Series:
+        mask = True * pd.DataFrame(True, shape=data.shape)
+        for function, args in mapping["filter"].items():
+            mask &= data.apply(function, arg)
+
+        return pd.Series()
+
     def _handle_extra_info(
         self,
         data: TabularData,
         table: str,
         col_def: Any,
         uuids: np.ndarray,
         extra_info: Optional[ExtraInfo],
+        table_mask
     ) -> None:
         """This function can extract extra info from the tabular data and store it in the extra_info dict
 
@@ -292,7 +309,7 @@ def _handle_extra_info(
         if extra_info is None:
             return
 
-        extra = self._parse_col_def(data=data, table=table, col_def=col_def, extra_info=None).to_dict(orient="records")
+        extra = self._parse_col_def(data=data, table=table, col_def=col_def, extra_info=None, table_mask=table_mask).to_dict(orient="records")
         for i, xtr in zip(uuids, extra):
             xtr = {
                 k[0] if isinstance(k, tuple) else k: v
@@ -339,7 +356,7 @@ def _serialize_data(self, data: Dataset, extra_info: Optional[ExtraInfo]) -> Tab
         return TabularData(logger=self._log, **data)
 
     def _parse_col_def(
-        self, data: TabularData, table: str, col_def: Any, extra_info: Optional[ExtraInfo]
+        self, data: TabularData, table: str, col_def: Any, extra_info: Optional[ExtraInfo], table_mask
     ) -> pd.DataFrame:
         """Interpret the column definition and extract/convert/create the data as a pandas DataFrame.
 
@@ -353,17 +370,17 @@ def _parse_col_def(
 
         """
         if isinstance(col_def, (int, float)):
-            return self._parse_col_def_const(data=data, table=table, col_def=col_def)
+            return self._parse_col_def_const(data=data, table=table, col_def=col_def, table_mask=table_mask)
         if isinstance(col_def, str):
-            return self._parse_col_def_column_name(data=data, table=table, col_def=col_def)
+            return self._parse_col_def_column_name(data=data, table=table, col_def=col_def, table_mask=table_mask)
         if isinstance(col_def, dict):
-            return self._parse_col_def_filter(data=data, table=table, col_def=col_def, extra_info=extra_info)
+            return self._parse_col_def_filter(data=data, table=table, col_def=col_def, extra_info=extra_info, table_mask=table_mask)
         if isinstance(col_def, list):
-            return self._parse_col_def_composite(data=data, table=table, col_def=col_def)
+            return self._parse_col_def_composite(data=data, table=table, col_def=col_def, table_mask=table_mask)
         raise TypeError(f"Invalid column definition: {col_def}")
 
     @staticmethod
-    def _parse_col_def_const(data: TabularData, table: str, col_def: Union[int, float]) -> pd.DataFrame:
+    def _parse_col_def_const(data: TabularData, table: str, col_def: Union[int, float], table_mask) -> pd.DataFrame:
         """Create a single column pandas DataFrame containing the const value.
 
         Args:
@@ -376,9 +393,9 @@ def _parse_col_def_const(data: TabularData, table: str, col_def: Union[int, floa
 
         """
         assert isinstance(col_def, (int, float))
-        return pd.DataFrame([col_def] * len(data[table]))
+        return pd.DataFrame([col_def] * len(data[table][table_mask]))
 
-    def _parse_col_def_column_name(self, data: TabularData, table: str, col_def: str) -> pd.DataFrame:
+    def _parse_col_def_column_name(self, data: TabularData, table: str, col_def: str, table_mask) -> pd.DataFrame:
         """Extract a column from the data. If the column doesn't exist, check if the col_def is a special float value,
         like 'inf'. If that's the case, create a single column pandas DataFrame containing the const value.
 
@@ -391,7 +408,7 @@ def _parse_col_def_column_name(self, data: TabularData, table: str, col_def: str
 
         """
         assert isinstance(col_def, str)
-        table_data = data[table]
+        table_data = data[table][table_mask]
 
         # If multiple columns are given in col_def, return the first column that exists in the dataset
         columns = [col_name.strip() for col_name in col_def.split("|")]
@@ -421,7 +438,7 @@ def _apply_multiplier(self, table: str, column: str, data: pd.Series) -> pd.Seri
             return data
 
     def _parse_reference(
-        self, data: TabularData, table: str, other_table: str, query_column: str, key_column: str, value_column: str
+        self, data: TabularData, table: str, other_table: str, query_column: str, key_column: str, value_column: str, table_mask
     ) -> pd.DataFrame:
         """
         Find and extract a column from a different table.
@@ -437,15 +454,15 @@ def _parse_reference(
         Returns:
 
         """
-        queries = self._parse_col_def_column_name(data=data, table=table, col_def=query_column)
-        keys = self._parse_col_def_column_name(data=data, table=other_table, col_def=key_column)
-        values = self._parse_col_def_column_name(data=data, table=other_table, col_def=value_column)
+        queries = self._parse_col_def_column_name(data=data, table=table, col_def=query_column, table_mask=table_mask)
+        keys = self._parse_col_def_column_name(data=data, table=other_table, col_def=key_column, table_mask=table_mask)
+        values = self._parse_col_def_column_name(data=data, table=other_table, col_def=value_column, table_mask=table_mask)
         other = pd.concat([keys, values], axis=1)
         result = queries.merge(other, how="left", left_on=query_column, right_on=key_column)
         return result[[value_column]]
 
     def _parse_col_def_filter(
-        self, data: TabularData, table: str, col_def: Dict[str, Any], extra_info: Optional[ExtraInfo]
+        self, data: TabularData, table: str, col_def: Dict[str, Any], extra_info: Optional[ExtraInfo], table_mask
     ) -> pd.DataFrame:
         """
         Parse column filters like 'auto_id', 'reference', 'function', etc
@@ -468,6 +485,7 @@ def _parse_col_def_filter(
                     ref_name=sub_def.get("name"),
                     key_col_def=sub_def["key"],
                     extra_info=extra_info,
+                    table_mask=table_mask,
                 )
             elif name == "reference":
                 # Check that (only) the required keys are in the definition
@@ -485,11 +503,12 @@ def _parse_col_def_filter(
                     query_column=sub_def["query_column"],
                     key_column=sub_def["key_column"],
                     value_column=sub_def["value_column"],
+                    table_mask=table_mask
                 )
             elif isinstance(sub_def, list):
-                col_data = self._parse_pandas_function(data=data, table=table, fn_name=name, col_def=sub_def)
+                col_data = self._parse_pandas_function(data=data, table=table, fn_name=name, col_def=sub_def, table_mask=table_mask)
             elif isinstance(sub_def, dict):
-                col_data = self._parse_function(data=data, table=table, function=name, col_def=sub_def)
+                col_data = self._parse_function(data=data, table=table, function=name, col_def=sub_def, table_mask=table_mask)
             else:
                 raise TypeError(f"Invalid {name} definition: {sub_def}")
             data_frames.append(col_data)
@@ -503,6 +522,7 @@ def _parse_auto_id(
         ref_name: Optional[str],
         key_col_def: Union[str, List[str], Dict[str, str]],
         extra_info: Optional[ExtraInfo],
+        table_mask,
     ) -> pd.DataFrame:
         """
         Create (or retrieve) a unique numerical id for each object (row) in `data[table]`, based on the `name`
@@ -517,7 +537,7 @@ def _parse_auto_id(
             key_col_def: A column definition which should be unique for each object within the current table
 
         Returns: A single column containing numerical ids
-
+    
         """
 
         # Handle reference table
@@ -535,7 +555,7 @@ def _parse_auto_id(
         else:
             raise TypeError(f"Invalid key definition type '{type(key_col_def).__name__}': {key_col_def}")
 
-        col_data = self._parse_col_def(data=data, table=table, col_def=key_col_def, extra_info=None)
+        col_data = self._parse_col_def(data=data, table=table, col_def=key_col_def, extra_info=None, table_mask=table_mask)
 
         def auto_id(row: np.ndarray):
             key = dict(zip(key_names, row))
@@ -558,7 +578,7 @@ def auto_id(row: np.ndarray):
 
         return col_data.apply(auto_id, axis=1, raw=True)
 
-    def _parse_pandas_function(self, data: TabularData, table: str, fn_name: str, col_def: List[Any]) -> pd.DataFrame:
+    def _parse_pandas_function(self, data: TabularData, table: str, fn_name: str, col_def: List[Any], table_mask) -> pd.DataFrame:
         """Special vectorized functions.
 
         Args:
@@ -576,7 +596,7 @@ def _parse_pandas_function(self, data: TabularData, table: str, fn_name: str, co
         if fn_name == "multiply":
             fn_name = "prod"
 
-        col_data = self._parse_col_def(data=data, table=table, col_def=col_def, extra_info=None)
+        col_data = self._parse_col_def(data=data, table=table, col_def=col_def, extra_info=None, table_mask=table_mask)
 
         try:
             fn_ptr = getattr(col_data, fn_name)
@@ -599,7 +619,7 @@ def _parse_pandas_function(self, data: TabularData, table: str, fn_name: str, co
 
         return pd.DataFrame(fn_ptr(axis=1))
 
-    def _parse_function(self, data: TabularData, table: str, function: str, col_def: Dict[str, Any]) -> pd.DataFrame:
+    def _parse_function(self, data: TabularData, table: str, function: str, col_def: Dict[str, Any], table_mask) -> pd.DataFrame:
         """Import the function by name and apply it to each row.
 
         Args:
@@ -616,15 +636,14 @@ def _parse_function(self, data: TabularData, table: str, function: str, col_def:
         fn_ptr = get_function(function)
         key_words = list(col_def.keys())
         sub_def = list(col_def.values())
-        col_data = self._parse_col_def(data=data, table=table, col_def=sub_def, extra_info=None)
+        col_data = self._parse_col_def(data=data, table=table, col_def=sub_def, extra_info=None, table_mask=table_mask)
 
         if col_data.empty:
             raise ValueError(f"Cannot apply function {function} to an empty DataFrame")
-
         col_data = col_data.apply(lambda row, fn=fn_ptr: fn(**dict(zip(key_words, row))), axis=1, raw=True)
         return pd.DataFrame(col_data)
 
-    def _parse_col_def_composite(self, data: TabularData, table: str, col_def: list) -> pd.DataFrame:
+    def _parse_col_def_composite(self, data: TabularData, table: str, col_def: list, table_mask) -> pd.DataFrame:
         """Select multiple columns (each is created from a column definition) and return them as a new DataFrame.
 
         Args:
@@ -636,7 +655,7 @@ def _parse_col_def_composite(self, data: TabularData, table: str, col_def: list)
 
         """
         assert isinstance(col_def, list)
-        columns = [self._parse_col_def(data=data, table=table, col_def=sub_def, extra_info=None) for sub_def in col_def]
+        columns = [self._parse_col_def(data=data, table=table, col_def=sub_def, extra_info=None, table_mask=table_mask) for sub_def in col_def]
         return pd.concat(columns, axis=1)
 
     def _get_id(self, table: str, key: Mapping[str, int], name: Optional[str]) -> int:

src/power_grid_model_io/functions/_functions.py

@@ -10,7 +10,7 @@
 
 import numpy as np
 import structlog
-from power_grid_model import WindingType
+from power_grid_model import MeasuredTerminalType, WindingType
 
 T = TypeVar("T")
 
@@ -104,3 +104,10 @@ def both_zeros_to_nan(value: float, other_value: float) -> float:
         _LOG.warning("0 replaced to nan")
         return float("nan")
     return value
+
+def filter_if_object(object_name: str, excl_object: str) -> bool:
+    """
+    Return false if the measured object should be excluded.
+    """
+    return object_name != excl_object
+

src/power_grid_model_io/functions/phase_to_phase.py

@@ -12,9 +12,34 @@
 
 from power_grid_model_io.functions import get_winding
 from power_grid_model_io.utils.parsing import parse_pvs_efficiency_type, parse_trafo3_connection, parse_trafo_connection
+from power_grid_model import MeasuredTerminalType
 
 _LOG = structlog.get_logger(__file__)
 
+MEASURED_TERMINAL_TYPE_MAP = {
+    "cable_from": MeasuredTerminalType.branch_from,
+    "cable_to": MeasuredTerminalType.branch_to,
+    "line_from": MeasuredTerminalType.branch_from,
+    "line_to": MeasuredTerminalType.branch_to,
+    "reactance_coil_from": MeasuredTerminalType.branch_from,
+    "reactance_coil_to": MeasuredTerminalType.branch_to,
+    "special_transformer_from": MeasuredTerminalType.branch_from,
+    "special_transformer_to": MeasuredTerminalType.branch_to,
+    "transformer_from": MeasuredTerminalType.branch_from,
+    "transformer_to": MeasuredTerminalType.branch_to,
+    "transformer_load": MeasuredTerminalType.branch_to,
+    "earthing_transformer": MeasuredTerminalType.branch_from,
+    "transformer3_1": MeasuredTerminalType.branch3_1,
+    "transformer3_2": MeasuredTerminalType.branch3_2,
+    "transformer3_3": MeasuredTerminalType.branch3_3,
+    "source": MeasuredTerminalType.source,
+    "shunt_capacitor": MeasuredTerminalType.shunt,
+    "shunt_reactor": MeasuredTerminalType.shunt,
+    "pv": MeasuredTerminalType.generator,
+    "wind_turbine": MeasuredTerminalType.generator,
+    "load": MeasuredTerminalType.load,
+}
+
 
 def relative_no_load_current(i_0: float, p_0: float, s_nom: float, u_nom: float) -> float:
     """
@@ -130,3 +155,14 @@ def pvs_power_adjustment(p: float, efficiency_type: str) -> float:
         return p * 0.95
 
     return p
+
+def get_measured_terminal_type(**kwargs) -> MeasuredTerminalType:
+    """
+    Return the measured terminal type, based on the string representation
+    """
+    for key, id in kwargs.items():
+        if id is not None:
+            return MEASURED_TERMINAL_TYPE_MAP[key]
+    _LOG.warning("No measured terminal type is found!")
+    return float("nan")
+

src/power_grid_model_io/utils/auto_id.py

@@ -119,4 +119,4 @@ def __getitem__(self, idx: int) -> Any:
         Returns:
             The original item
         """
-        return self._items[idx]
+        return self._items.get(idx)