Merge pull request #6906 from janezd/groupby-keep-variables

GroupBy: Avoid guessing variable types

Author: janezd

Orange/data/aggregate.py

@@ -1,5 +1,5 @@
 from functools import lru_cache
-from typing import Callable, Dict, List, Tuple, Union
+from typing import Callable, Dict, List, Tuple, Union, Type
 
 import pandas as pd
 
@@ -39,15 +39,20 @@ def __init__(self, table: Table, by: List[Variable]):
         df = table_to_frame(table, include_metas=True)
         # observed=True keeps only groups with at leas one instance
         self.group_by = df.groupby([a.name for a in by], observed=True)
+        self.by = tuple(by)
 
         # lru_cache that is caches on the object level
         self.compute_aggregation = lru_cache()(self._compute_aggregation)
 
+    AggDescType = Union[str,
+                    Callable,
+                    Tuple[str, Union[str, Callable]],
+                    Tuple[str, Union[str, Callable], Union[Type[Variable], bool]]
+    ]
+
     def aggregate(
         self,
-        aggregations: Dict[
-            Variable, List[Union[str, Callable, Tuple[str, Union[str, Callable]]]]
-        ],
+        aggregations: Dict[Variable, List[AggDescType]],
         callback: Callable = dummy_callback,
     ) -> Table:
         """
@@ -57,12 +62,16 @@ def aggregate(
         ----------
         aggregations
             The dictionary that defines aggregations that need to be computed
-            for variables. We support two formats:
+            for variables. We support three formats:
             - {variable name: [agg function 1, agg function 2]}
             - {variable name: [(agg name 1, agg function 1),  (agg name 1, agg function 1)]}
+            - {variable name: [(agg name 1, agg function 1, output_variable_type1), ...]}
             Where agg name is the aggregation name used in the output column name.
             Aggregation function can be either function or string that defines
             aggregation in Pandas (e.g. mean).
+            output_variable_type can be a type for a new variable, True to copy
+            the input variable, or False to create a new variable of the same type
+            as the input
         callback
             Callback function to report the progress
 
@@ -75,29 +84,44 @@ def aggregate(
         count = 0
 
         result_agg = []
+        output_variables = []
         for col, aggs in aggregations.items():
             for agg in aggs:
-                res = self._compute_aggregation(col, agg)
+                res, var = self._compute_aggregation(col, agg)
                 result_agg.append(res)
+                output_variables.append(var)
                 count += 1
                 callback(count / num_aggs * 0.8)
 
-        agg_table = self._aggregations_to_table(result_agg)
+        agg_table = self._aggregations_to_table(result_agg, output_variables)
         callback(1)
         return agg_table
 
     def _compute_aggregation(
-        self, col: Variable, agg: Union[str, Callable, Tuple[str, Union[str, Callable]]]
-    ) -> pd.Series:
+            self, col: Variable, agg: AggDescType) -> Tuple[pd.Series, Variable]:
         # use named aggregation to avoid issues with same column names when reset_index
         if isinstance(agg, tuple):
-            name, agg = agg
+            name, agg, var_type, *_ = (*agg, None)
         else:
             name = agg if isinstance(agg, str) else agg.__name__
+            var_type = None
         col_name = f"{col.name} - {name}"
-        return self.group_by[col.name].agg(**{col_name: agg})
-
-    def _aggregations_to_table(self, aggregations: List[pd.Series]) -> Table:
+        agg_col = self.group_by[col.name].agg(**{col_name: agg})
+        if var_type is True:
+            var = col.copy(name=col_name)
+        elif var_type is False:
+            var = col.make(name=col_name)
+        elif var_type is None:
+            var = None
+        else:
+            assert issubclass(var_type, Variable)
+            var = var_type.make(name=col_name)
+        return agg_col, var
+
+    def _aggregations_to_table(
+            self,
+            aggregations: List[pd.Series],
+            output_variables: List[Union[Variable, None]]) -> Table:
         """Concatenate aggregation series and convert back to Table"""
         if aggregations:
             df = pd.concat(aggregations, axis=1)
@@ -107,7 +131,7 @@ def _aggregations_to_table(self, aggregations: List[pd.Series]) -> Table:
             df = df.drop(columns=df.columns)
         gb_attributes = df.index.names
         df = df.reset_index()  # move group by var that are in index to columns
-        table = table_from_frame(df)
+        table = table_from_frame(df, variables=(*self.by, *output_variables))
 
         # group by variables should be last two columns in metas in the output
         metas = table.domain.metas

Orange/data/pandas_compat.py

@@ -1,5 +1,6 @@
 """Pandas DataFrame↔Table conversion helpers"""
 from functools import partial
+from itertools import zip_longest
 
 import numpy as np
 from scipy import sparse as sp
@@ -255,7 +256,14 @@ def to_categorical(s, _):
     return np.asarray(x)
 
 
-def vars_from_df(df, role=None, force_nominal=False):
+def to_numeric(s, _):
+    return np.asarray(pd.to_numeric(s))
+
+
+def vars_from_df(df, role=None, force_nominal=False, variables=None):
+    if variables is not None:
+        assert len(variables) == len(df.columns)
+
     if role is None and hasattr(df, 'orange_role'):
         role = df.orange_role
     df = _reset_index(df)
@@ -264,39 +272,52 @@ def vars_from_df(df, role=None, force_nominal=False):
     exprs = [], [], []
     vars_ = [], [], []
 
-    for column in df.columns:
+    def _convert_string(s, _):
+        return np.asarray(
+                    # to object so that fillna can replace with nans if Unknown in nan
+                    # replace nan with object Unknown assure that all values are string
+                    s.astype(object).fillna(StringVariable.Unknown).astype(str),
+                    dtype=object
+                )
+
+    conversions = {
+        DiscreteVariable: to_categorical,
+        ContinuousVariable: to_numeric,
+        TimeVariable: _convert_datetime,
+        StringVariable: _convert_string
+    }
+
+    for column, var in zip_longest(df.columns, variables or [], fillvalue=None):
         s = df[column]
         _role = Role.Attribute if role is None else role
-        if hasattr(df, 'orange_variables') and column in df.orange_variables:
+        if var is not None:
+            if not var.is_primitive():
+                _role = Role.Meta
+            expr = conversions[type(var)]
+        elif hasattr(df, 'orange_variables') and column in df.orange_variables:
             original_var = df.orange_variables[column]
             var = original_var.copy(compute_value=None)
             expr = None
-        elif _is_datetime(s):
-            var = TimeVariable(str(column))
-            expr = _convert_datetime
-        elif _is_discrete(s, force_nominal):
-            discrete = s.astype("category").cat
-            var = DiscreteVariable(
-                str(column), discrete.categories.astype(str).tolist()
-            )
-            expr = to_categorical
-        elif is_numeric_dtype(s):
-            var = ContinuousVariable(
-                # set number of decimals to 0 if int else keeps default behaviour
-                str(column), number_of_decimals=(0 if is_integer_dtype(s) else None)
-            )
-            expr = None
         else:
-            if role is not None and role != Role.Meta:
-                raise ValueError("String variable must be in metas.")
-            _role = Role.Meta
-            var = StringVariable(str(column))
-            expr = lambda s, _: np.asarray(
-                # to object so that fillna can replace with nans if Unknown in nan
-                # replace nan with object Unknown assure that all values are string
-                s.astype(object).fillna(StringVariable.Unknown).astype(str),
-                dtype=object
-            )
+            if _is_datetime(s):
+                var = TimeVariable(str(column))
+            elif _is_discrete(s, force_nominal):
+                discrete = s.astype("category").cat
+                var = DiscreteVariable(
+                    str(column), discrete.categories.astype(str).tolist()
+                )
+            elif is_numeric_dtype(s):
+                var = ContinuousVariable(
+                    # set number of decimals to 0 if int else keeps default behaviour
+                    str(column), number_of_decimals=(0 if is_integer_dtype(s) else None)
+                )
+            else:
+                if role is not None and role != Role.Meta:
+                    raise ValueError("String variable must be in metas.")
+                _role = Role.Meta
+                var = StringVariable(str(column))
+            expr = conversions[type(var)]
+
 
         cols[_role].append(column)
         exprs[_role].append(expr)
@@ -330,8 +351,10 @@ def vars_from_df(df, role=None, force_nominal=False):
     return xym, Domain(*vars_)
 
 
-def table_from_frame(df, *, force_nominal=False):
-    XYM, domain = vars_from_df(df, force_nominal=force_nominal)
+def table_from_frame(df, *, force_nominal=False, variables=None):
+    XYM, domain = vars_from_df(df,
+                               force_nominal=force_nominal,
+                               variables=variables)
 
     if hasattr(df, 'orange_weights') and hasattr(df, 'orange_attributes'):
         W = [df.orange_weights[i] for i in df.index if i in df.orange_weights]

Orange/data/tests/test_aggregate.py

@@ -1,4 +1,5 @@
 import unittest
+from unittest.mock import Mock
 
 import numpy as np
 import pandas as pd
@@ -132,13 +133,69 @@ def test_aggregation(self):
     def test_preserve_table_class(self):
         """
         Test whether result table has the same type than the imnput table,
-        e.g. if input table corpus the resutlitn table must be corpus too.
+        e.g. if input table corpus the resulting table must be corpus too.
         """
         data = AlternativeTable.from_table(self.data.domain, self.data)
         gb = data.groupby([data.domain["a"]])
         output = gb.aggregate({data.domain["a"]: ["mean"]})
         self.assertIsInstance(output, AlternativeTable)
 
+    def test_preserve_variables(self):
+        a, _, _, dvar = self.data.domain.attributes
+        gb = self.data.groupby([a])
+
+        a.attributes = {"foo": "bar"}
+        dvar.attributes = {"foo": "baz"}
+
+        a.copy = Mock(side_effect=a.copy)
+        a.make = Mock(side_effect=a.make)
+
+        def f(*_):
+            return 0
+
+        output = gb.aggregate(
+            {a: [("copy", f, True),
+                 ("make", f, False),
+                 ("auto", f, None),
+                 ("string", f, StringVariable),
+                 ("number", f, ContinuousVariable)],
+             dvar: [("copy", f, True),
+                    ("make", f, False),
+                    ("auto", f, None),
+                    ("string", f, StringVariable),
+                    ("discrete", f, DiscreteVariable)]}
+        )
+        self.assertIsInstance(output.domain["a - copy"], ContinuousVariable)
+        a.copy.assert_called_once()
+        self.assertEqual(output.domain["a - copy"].attributes, {"foo": "bar"})
+
+        self.assertIsInstance(output.domain["a - make"], ContinuousVariable)
+        a.make.assert_called_once()
+        self.assertNotEqual(output.domain["a - make"].attributes, {"foo": "bar"})
+
+        self.assertIsInstance(output.domain["a - auto"], ContinuousVariable)
+        self.assertNotEqual(output.domain["a - auto"].attributes, {"foo": "bar"})
+
+        self.assertIsInstance(output.domain["a - string"], StringVariable)
+
+        self.assertIsInstance(output.domain["a - number"], ContinuousVariable)
+        self.assertNotEqual(output.domain["a - number"].attributes, {"foo": "bar"})
+
+        self.assertIsInstance(output.domain["dvar - copy"], DiscreteVariable)
+        self.assertEqual(output.domain["dvar - copy"].attributes, {"foo": "baz"})
+
+        self.assertIsInstance(output.domain["dvar - make"], DiscreteVariable)
+        self.assertNotEqual(output.domain["dvar - make"].attributes, {"foo": "baz"})
+
+        # f returns 0, so the column looks numeric! Let's test that it is
+        # converted to numeric.
+        self.assertIsInstance(output.domain["dvar - auto"], ContinuousVariable)
+
+        self.assertIsInstance(output.domain["dvar - string"], StringVariable)
+
+        self.assertIsInstance(output.domain["dvar - discrete"], DiscreteVariable)
+        self.assertNotEqual(output.domain["dvar - discrete"].attributes, {"foo": "baz"})
+
 
 if __name__ == "__main__":
     unittest.main()

Orange/data/tests/test_pandas.py Orange/data/tests/test_pandas_compat.pyOrange/data/tests/test_pandas.py renamed to Orange/data/tests/test_pandas_compat.py

@@ -56,6 +56,24 @@ def test_table_from_frame(self):
         self.assertEqual(names, ['0', '1', '2'])
         self.assertEqual(types, [DiscreteVariable, ContinuousVariable, TimeVariable])
 
+        # Specify (some) variables
+        dvar = DiscreteVariable('x', values=tuple("dacb"))
+        cvar = ContinuousVariable('y')
+        table = table_from_frame(df, variables=[dvar, cvar, None])
+        self.assertIs(table.domain[0], dvar)
+        self.assertIs(table.domain[1], cvar)
+        self.assertIsInstance(table.domain[2], TimeVariable)
+
+        table = table_from_frame(df,
+                                 variables=[None, None, None],
+                                 force_nominal=True)
+        self.assertIsInstance(table.domain[0], DiscreteVariable)
+        self.assertIsInstance(table.domain[1], ContinuousVariable)
+        self.assertIsInstance(table.domain[2], TimeVariable)
+
+        self.assertRaises(AssertionError,
+                          table_from_frame, df, variables=[None, None])
+
         # Include index
         df.index = list('abaa')
         table = table_from_frame(df)