Merge pull request #124 from FAST-HEP/BK_fix_binned_df_empty

Handle empty data chunks in binned df explode

Author: benkrikler

.travis.yml

@@ -2,10 +2,9 @@ dist: xenial
 language: python
 
 python:
- - "2.7"
- - "3.5"
  - "3.6"
  - "3.7"
+ - "3.8"
 
 install:
   - pip install -r .requirements_dev.txt
@@ -32,4 +31,4 @@ deploy:
   on:
     tags: true
     repo: FAST-HEP/fast-carpenter
-    condition: "$TRAVIS_PYTHON_VERSION == 3.6 && $TRAVIS_TAG =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+|[.]dev[0-9]+)?$"
+    condition: "$TRAVIS_PYTHON_VERSION == 3.7 && $TRAVIS_TAG =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+|[.]dev[0-9]+)?$"

CHANGELOG.md

@@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Removed
+- Testing against Python <= 3.5, PR #124
+
+### Fixed
+- Fix handling of empty data chunks in BinnedDataframe stage, PR #124 [@BenKrikler](https://github.com/benkrikler)
 
 ## [0.17.5] - 2020-04-03
 ### Added

fast_carpenter/summary/binned_dataframe.py

@@ -70,12 +70,18 @@ def _merge_dataframes(dataset_readers_list):
     for dataset, readers in dataset_readers_list:
         dataset_df = readers[0]
         for df in readers[1:]:
-            if df is None:
+            if df is None or df.empty:
                 continue
             dataset_df = dataset_df.add(df, fill_value=0.)
+        if dataset_df is None or dataset_df.empty:
+            continue
         all_dfs.append(dataset_df)
         keys.append(dataset)
-    final_df = pd.concat(all_dfs, keys=keys, names=['dataset'], sort=True)
+    if all_dfs:
+        final_df = pd.concat(all_dfs, keys=keys, names=['dataset'], sort=True)
+    else:
+        final_df = pd.DataFrame()
+
     return final_df
 
 
@@ -204,6 +210,8 @@ def event(self, chunk):
 
         data = chunk.tree.pandas.df(all_inputs, flatten=False)
         data = explode(data)
+        if data is None or data.empty:
+            return True
 
         binned_values = _bin_values(data, dimensions=self._bin_dims,
                                     binnings=self._binnings,
@@ -279,6 +287,9 @@ def explode(df):
     https://stackoverflow.com/questions/12680754/split-explode-pandas\
     -dataframe-string-entry-to-separate-rows/40449726#40449726
     """
+    if df is None or df.empty:
+        return df
+
     # get the list columns
     lst_cols = [col for col, dtype in df.dtypes.items() if is_object_dtype(dtype)]
     # Be more specific about which objects are ok

tests/summary/test_binned_dataframe.py

@@ -258,6 +258,9 @@ def test_explode():
     assert len(exploded) == 1 + 8 + 3
     assert np.array_equal(exploded.list, [0, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2])
 
+    exploded = bdf.explode(pd.DataFrame(columns=["one", "two", "3"]))
+    assert exploded.empty is True
+
 
 def test_densify_dataframe_integers():
     index = [("one", 1), ("one", 3), ("two", 2), ("three", 1), ("three", 2)]