Fix memory error when computing unique values (#313)

Author: andersy005

intake_esm/search.py

@@ -12,11 +12,16 @@ def _unique(df, columns=None):
         columns = [columns]
     if not columns:
         columns = df.columns.tolist()
+
+    def _find_unique(series):
+        values = series.dropna().values
+        uniques = list(set(_flatten_list(values)))
+        return uniques
+
+    x = df[columns].apply(_find_unique, result_type='reduce').to_dict()
     info = {}
-    for col in columns:
-        values = df[col].dropna().values
-        uniques = np.unique(list(_flatten_list(values))).tolist()
-        info[col] = {'count': len(uniques), 'values': uniques}
+    for col in x.keys():
+        info[col] = {'count': len(x[col]), 'values': x[col]}
     return info
 
 

tests/test_search.py

@@ -32,14 +32,17 @@ def test_unique():
         'random': {'count': 3, 'values': ['bx', 'by', 'bz']},
     }
     actual = _unique(df, df.columns.tolist())
-    assert actual == expected
+
+    assert set(actual.keys()) == set(expected.keys())
+    for key in actual.keys():
+        assert set(actual[key]['values']) == set(expected[key]['values'])
 
     actual = _unique(df)
-    assert actual == expected
+    assert set(actual.keys()) == set(expected.keys())
 
     actual = _unique(df, columns='random')
     expected = {'random': {'count': 3, 'values': ['bx', 'by', 'bz']}}
-    assert actual == expected
+    assert set(actual['random']['values']) == set(expected['random']['values'])
 
 
 params = [