Merge branch 'main' into feature/ohe_values_2

Author: Boluwatife28

CHANGELOG.rst

@@ -37,13 +37,21 @@ Changed
 - narwhalified BaseNumericTransformer `#358 https://github.com/lvgig/tubular/issues/358`
 - narwhalified DropOriginalMixin `#352 <https://github.com/lvgig/tubular/issues/352>_`
 - narwhalified BaseMappingTransformer `#367 <https://github.com/lvgig/tubular/issues/367>_`
+- narwhalified BaseMappingTransformerMixin. As part of this made mapping transformers more 
+type-conscious, they now rely on an input 'return_dtypes' dict arg.
+`#369 <https://github.com/lvgig/tubular/issues/369>_`
+- As part of #369, updated OrdinalEncoderTransformer to output Int8 type
+- As part of #369, updated NominalToIntegerTransformer to output Int8 type. Removed inverse_mapping
+functionality, as this is more complicated when transform is opinionated on types.
+- narwhalified GroupRareLevelsTransformer. As part of this, had to make transformer more opinionated
+and refuse columns with nulls (raises an error directing to imputers.) `#372 <https://github.com/lvgig/tubular/issues/372>_`
 - narwhalified BaseDatetimeTransformer `#375 <https://github.com/azukds/tubular/issues/375>`
 - Optional wanted_levels feature has been integrated into the OneHotEncodingTransformer which allows users to specify which levels in a column they wish to encode. `#384 <https://github.com/azukds/tubular/issues/384>_`
 - Created unit tests to check if the values provided for wanted_values are as expected and if the output is as expected.
 - placeholder
 - placeholder
-
-
+- placeholder
+- placeholder
 
 1.4.1 (02/12/2024)
 ------------------

pyproject.toml

@@ -18,8 +18,9 @@ dynamic = ["version"]
 dependencies = [
     "pandas>=1.5.0",
     "scikit-learn>=1.2.0",
-    "narwhals >= 1.17.0",
+    "narwhals >= 1.21.1",
     "polars >= 1.9.0",
+    "beartype >= 0.19.0",
 ]
 requires-python = ">=3.9"
 authors = [{ name = "LV GI Data Science Team", email="#[email protected]"}]
@@ -130,6 +131,8 @@ dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 "__init__.py" = ["E402", "F401"]
 "tests/*" = ["ANN", "S101"]
 
+[tool.ruff.lint.pyupgrade]
+keep-runtime-typing=true
 
 [tool.coverage.run]
 branch = true

requirements-dev.txt

@@ -4,6 +4,8 @@
 #
 #    pip-compile --extra=dev --no-emit-index-url --no-emit-trusted-host --output-file=requirements-dev.txt pyproject.toml
 #
+beartype==0.19.0
+    # via tubular (pyproject.toml)
 cfgv==3.4.0
     # via pre-commit
 coverage[toml]==7.6.0
@@ -20,7 +22,7 @@ iniconfig==2.0.0
     # via pytest
 joblib==1.4.2
     # via scikit-learn
-narwhals==1.17.0
+narwhals==1.21.1
     # via tubular (pyproject.toml)
 nodeenv==1.9.1
     # via pre-commit

tests/base_tests.py

@@ -453,7 +453,7 @@ def test_unexpected_kwarg_error(
             ),
         ):
             uninitialized_transformers[self.transformer_name](
-                unexpected_kwarg="spanish inquisition",
+                unexpected_kwarg=True,
                 **minimal_attribute_dict[self.transformer_name],
             )
 

tests/conftest.py

@@ -127,6 +127,9 @@ def minimal_attribute_dict():
         },
         "BaseMappingTransformer": {
             "mappings": {"a": {1: 2, 3: 4}},
+            # this arg is dependent on inputs, so
+            # may not generalise well
+            "return_dtypes": {"a": "Int8"},
         },
         "BaseMappingTransformMixin": {
             "columns": ["a"],
@@ -208,6 +211,9 @@ def minimal_attribute_dict():
         },
         "MappingTransformer": {
             "mappings": {"a": {1: 2, 3: 4}},
+            # this arg is dependent on inputs, so
+            # may not generalise well
+            "return_dtypes": {"a": "Int8"},
         },
         "MeanImputer": {
             "columns": ["b"],

tests/mapping/test_BaseCrossColumnMappingTransformer.py

@@ -31,6 +31,35 @@ def test_adjust_columns_non_string_error(
         ):
             uninitialized_transformers[self.transformer_name](**args)
 
+    def test_inferred_return_dtypes(
+        self,
+        uninitialized_transformers,
+        minimal_attribute_dict,
+    ):
+        """test that return_dtypes are inferred correctly if not provided - test is
+        overloaded as these transformers require OrderedDict for multicolumn mapping
+        """
+
+        kwargs = minimal_attribute_dict[self.transformer_name]
+        kwargs["mappings"] = {
+            "a": {"a": 1, "b": 2},
+        }
+        kwargs["return_dtypes"] = None
+
+        transformer = uninitialized_transformers[self.transformer_name](
+            **kwargs,
+        )
+
+        expected = {
+            "a": "Int64",
+        }
+
+        actual = transformer.return_dtypes
+
+        assert (
+            actual == expected
+        ), f"return_dtypes attr not inferred as expected, expected {expected} but got {actual}"
+
 
 class BaseCrossColumnMappingTransformerTransformTests(
     BaseMappingTransformerTransformTests,

tests/mapping/test_BaseCrossColumnNumericTransformer.py

@@ -36,6 +36,37 @@ def test_mapping_values_not_numeric_error(
         ):
             uninitialized_transformers[self.transformer_name](**args)
 
+    def test_inferred_return_dtypes(
+        self,
+        uninitialized_transformers,
+        minimal_attribute_dict,
+    ):
+        """test that return_dtypes are inferred correctly if not provided - test is
+        overloaded as these transformers can only handle numeric types
+        """
+
+        kwargs = minimal_attribute_dict[self.transformer_name]
+        kwargs["mappings"] = {
+            "a": {"a": 1, "b": 2},
+            "c": {"d": 1.0, "e": 2.0},
+        }
+        kwargs["return_dtypes"] = None
+
+        transformer = uninitialized_transformers[self.transformer_name](
+            **kwargs,
+        )
+
+        expected = {
+            "a": "Int64",
+            "c": "Float64",
+        }
+
+        actual = transformer.return_dtypes
+
+        assert (
+            actual == expected
+        ), f"return_dtypes attr not inferred as expected, expected {expected} but got {actual}"
+
 
 class BaseCrossColumnNumericTransformerTransformTests(
     BaseCrossColumnMappingTransformerTransformTests,

tests/mapping/test_BaseMappingTransformMixin.py

@@ -2,8 +2,8 @@
 import re
 
 import pandas as pd
+import polars as pl
 import pytest
-import test_aide as ta
 
 import tests.test_data as d
 from tests.base_tests import (
@@ -12,7 +12,7 @@
     GenericTransformTests,
     OtherBaseBehaviourTests,
 )
-from tests.utils import assert_frame_equal_dispatch
+from tests.utils import assert_frame_equal_dispatch, dataframe_init_dispatch
 from tubular.mapping import BaseMappingTransformMixin
 
 # Note there are no tests that need inheriting from this file as the only difference is an expected transform output
@@ -21,8 +21,8 @@
 @pytest.fixture()
 def mapping():
     return {
-        "a": {1: "a", 2: "b", 3: "c", 4: "d", 5: "e", 6: "f"},
-        "b": {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6},
+        "a": {1: "a", 2: "b", 3: "c", 4: "d", 5: "e", 6: "f", 7: "g", 8: "h", 9: None},
+        "b": {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, None: 9},
     }
 
 
@@ -51,100 +51,133 @@ class TestTransform(GenericTransformTests):
     def setup_class(cls):
         cls.transformer_name = "BaseMappingTransformMixin"
 
-    def test_expected_output(self, mapping):
+    @pytest.mark.parametrize("library", ["pandas", "polars"])
+    def test_expected_output(self, mapping, library):
         """Test that X is returned from transform."""
 
-        df = d.create_df_1()
+        df = d.create_df_1(library=library)
 
-        expected = pd.DataFrame(
-            {
-                "a": ["a", "b", "c", "d", "e", "f"],
-                "b": [1, 2, 3, 4, 5, 6],
-            },
+        expected_dict = {
+            "a": ["a", "b", "c", "d", "e", "f"],
+            "b": [1, 2, 3, 4, 5, 6],
+        }
+
+        expected = dataframe_init_dispatch(
+            dataframe_dict=expected_dict,
+            library=library,
         )
 
-        x = BaseMappingTransformMixin(columns=["a", "b"])
+        transformer = BaseMappingTransformMixin(columns=["a", "b"])
 
-        x.mappings = mapping
+        # if transformer is not yet polars compatible, skip this test
+        if not transformer.polars_compatible and isinstance(df, pl.DataFrame):
+            return
 
-        df_transformed = x.transform(df)
+        transformer.mappings = mapping
+        transformer.return_dtypes = {"a": "String", "b": "Int64"}
 
-        ta.equality.assert_equal_dispatch(
-            expected=expected,
-            actual=df_transformed,
-            msg="BaseMappingTransformMixin from transform",
-        )
+        df_transformed = transformer.transform(df)
+
+        assert_frame_equal_dispatch(expected, df_transformed)
 
-    def test_mappings_unchanged(self, mapping):
+    @pytest.mark.parametrize("library", ["pandas", "polars"])
+    def test_mappings_unchanged(self, mapping, library):
         """Test that mappings is unchanged in transform."""
-        df = d.create_df_1()
+        df = d.create_df_1(library=library)
 
-        x = BaseMappingTransformMixin(columns=["a", "b"])
+        transformer = BaseMappingTransformMixin(columns=["a", "b"])
 
-        x.mappings = mapping
+        # if transformer is not yet polars compatible, skip this test
+        if not transformer.polars_compatible and isinstance(df, pl.DataFrame):
+            return
 
-        x.transform(df)
+        transformer.mappings = mapping
+        transformer.return_dtypes = {
+            "a": "String",
+            "b": "Int64",
+        }
 
-        ta.equality.assert_equal_dispatch(
-            expected=mapping,
-            actual=x.mappings,
-            msg="BaseMappingTransformer.transform has changed self.mappings unexpectedly",
-        )
+        transformer.transform(df)
+
+        assert (
+            mapping == transformer.mappings
+        ), f"BaseMappingTransformer.transform has changed self.mappings unexpectedly, expected {mapping} but got {transformer.mappings}"
 
+    @pytest.mark.parametrize("library", ["pandas", "polars"])
     @pytest.mark.parametrize("non_df", [1, True, "a", [1, 2], {"a": 1}, None])
     def test_non_pd_type_error(
         self,
         non_df,
         mapping,
+        library,
     ):
         """Test that an error is raised in transform is X is not a pd.DataFrame."""
 
-        df = d.create_df_10()
+        df = d.create_df_10(library=library)
 
-        x = BaseMappingTransformMixin(columns=["a"])
+        transformer = BaseMappingTransformMixin(columns=["a"])
 
-        x.mappings = mapping
+        # if transformer is not yet polars compatible, skip this test
+        if not transformer.polars_compatible and isinstance(df, pl.DataFrame):
+            return
 
-        x_fitted = x.fit(df, df["c"])
+        transformer.mappings = mapping
+        transformer.return_dtypes = {
+            "a": "String",
+        }
+
+        x_fitted = transformer.fit(df, df["c"])
 
         with pytest.raises(
             TypeError,
             match="BaseMappingTransformMixin: X should be a polars or pandas DataFrame/LazyFrame",
         ):
             x_fitted.transform(X=non_df)
 
-    def test_no_rows_error(self, mapping):
+    @pytest.mark.parametrize("library", ["pandas", "polars"])
+    def test_no_rows_error(self, mapping, library):
         """Test an error is raised if X has no rows."""
-        df = d.create_df_10()
+        df = d.create_df_10(library=library)
+
+        transformer = BaseMappingTransformMixin(columns=["a"])
 
-        x = BaseMappingTransformMixin(columns=["a"])
+        # if transformer is not yet polars compatible, skip this test
+        if not transformer.polars_compatible and isinstance(df, pl.DataFrame):
+            return
 
-        x.mappings = mapping
+        transformer.mappings = mapping
+        transformer.return_dtypes = {"a": "String"}
 
-        x = x.fit(df, df["c"])
+        transformer = transformer.fit(df, df["c"])
 
         df = pd.DataFrame(columns=["a", "b", "c"])
 
         with pytest.raises(
             ValueError,
             match=re.escape("BaseMappingTransformMixin: X has no rows; (0, 3)"),
         ):
-            x.transform(df)
+            transformer.transform(df)
 
-    def test_original_df_not_updated(self, mapping):
+    @pytest.mark.parametrize("library", ["pandas", "polars"])
+    def test_original_df_not_updated(self, mapping, library):
         """Test that the original dataframe is not transformed when transform method used."""
 
-        df = d.create_df_10()
+        df = d.create_df_10(library=library)
+
+        transformer = BaseMappingTransformMixin(columns=["a"])
 
-        x = BaseMappingTransformMixin(columns=["a"])
+        # if transformer is not yet polars compatible, skip this test
+        if not transformer.polars_compatible and isinstance(df, pl.DataFrame):
+            return
 
-        x.mappings = mapping
+        transformer.mappings = mapping
+        transformer.return_dtypes = {"a": "String", "b": "Int64"}
 
-        x = x.fit(df, df["c"])
+        transformer = transformer.fit(df, df["c"])
 
-        _ = x.transform(df)
+        _ = transformer.transform(df)
 
-        pd.testing.assert_frame_equal(df, d.create_df_10())
+        assert_frame_equal_dispatch(df, d.create_df_10(library=library))
 
     @pytest.mark.parametrize(
         "minimal_dataframe_lookup",
@@ -160,17 +193,23 @@ def test_pandas_index_not_updated(
         """Test that the original (pandas) dataframe index is not transformed when transform method used."""
 
         df = minimal_dataframe_lookup[self.transformer_name]
-        x = initialized_transformers[self.transformer_name]
-        x.mappings = mapping
+        transformer = initialized_transformers[self.transformer_name]
+
+        # if transformer is not yet polars compatible, skip this test
+        if not transformer.polars_compatible and isinstance(df, pl.DataFrame):
+            return
+
+        transformer.mappings = mapping
+        transformer.return_dtypes = {"a": "String", "b": "String"}
 
         # update to abnormal index
         df.index = [2 * i for i in df.index]
 
         original_df = copy.deepcopy(df)
 
-        x = x.fit(df, df["a"])
+        transformer = transformer.fit(df, df["a"])
 
-        _ = x.transform(df)
+        _ = transformer.transform(df)
 
         assert_frame_equal_dispatch(df, original_df)