Fit_transform all transformers iteratively when transformers is a list.

Author: dukebody

sklearn_pandas/__init__.py

@@ -122,8 +122,12 @@ def fit(self, X, y=None):
         for columns, transformers in self.features:
             if transformers is not None:
                 if isinstance(transformers, list):
-                    for transformer in transformers:
-                        transformer.fit(self._get_col_subset(X, columns))
+                    # first fit_transform all transformers except the last one
+                    Xt = self._get_col_subset(X, columns)
+                    for transformer in transformers[:-1]:
+                        Xt = transformer.fit_transform(Xt)
+                    # then fit the last one without transformation
+                    transformers[-1].fit(Xt)
                 else:
                     transformers.fit(self._get_col_subset(X, columns))
         return self
@@ -139,14 +143,14 @@ def transform(self, X):
             # columns could be a string or list of
             # strings; we don't care because pandas
             # will handle either.
-            final_feature = self._get_col_subset(X, columns)
+            Xt = self._get_col_subset(X, columns)
             if transformers is not None:
                 if isinstance(transformers, list):
                     for transformer in transformers:
-                        final_feature = transformer.transform(final_feature)
+                        Xt = transformer.transform(Xt)
                 else:
-                    final_feature = transformers.transform(final_feature)
-            extracted.append(_handle_feature(final_feature))
+                    Xt = transformers.transform(Xt)
+            extracted.append(_handle_feature(Xt))
 
         # combine the feature outputs into one array.
         # at this point we lose track of which features

tests/test_dataframe_mapper.py

@@ -6,6 +6,7 @@
 from sklearn.pipeline import Pipeline
 from sklearn.svm import SVC
 from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.preprocessing import Imputer, StandardScaler
 import numpy as np
 
 from sklearn_pandas import (
@@ -54,11 +55,27 @@ def test_with_iris_dataframe(iris_dataframe):
 def test_with_car_dataframe(cars_dataframe):
     pipeline = Pipeline([
         ("preprocess", DataFrameMapper([
-            ("description", [PassthroughTransformer(), CountVectorizer()]),
+            ("description", CountVectorizer()),
         ])),
         ("classify", SVC(kernel='linear'))
     ])
     data = cars_dataframe.drop("model", axis=1)
     labels = cars_dataframe["model"]
     scores = cross_val_score(pipeline, data, labels)
     assert scores.mean() > 0.30
+
+
+def test_list_transformers():
+    dataframe = pd.DataFrame({"a": [1, np.nan, 3], "b": [1, 5, 7]})
+
+    mapper = DataFrameMapper([
+        (["a"], [Imputer(), StandardScaler()]),
+        (["b"], StandardScaler()),
+    ])
+    dmatrix = mapper.fit_transform(dataframe)
+
+    assert pd.isnull(dmatrix).sum() == 0  # no null values
+
+    # all features have mean 0 and std deviation 1 (standardized)
+    assert (abs(dmatrix.mean(axis=0) - 0) <= 1e-6).all()
+    assert (abs(dmatrix.std(axis=0) - 1) <= 1e-6).all()