ENH: Add incremental algorithms support

configs/README.md

@@ -117,6 +117,7 @@ Configs have the three highest parameter keys:
 |:---------------|:--------------|:--------|:------------|
 | `algorithm`:`estimator` | None |  | Name of measured estimator. |
 | `algorithm`:`estimator_params` | Empty `dict` |  | Parameters for estimator constructor. |
+| `algorithm`:`training`:`num_batches` | 5 |  | Number of batches to benchmark `partial_fit` function, using batches the size of number of samples specified (not samples divided by `num_batches`). For incremental estimators only. |
 | `algorithm`:`online_inference_mode` | False |  | Enables online mode for inference methods of estimator (separate call for each sample). |
 | `algorithm`:`sklearn_context` | None |  | Parameters for sklearn `config_context` used over estimator. |
 | `algorithm`:`sklearnex_context` | None |  | Parameters for sklearnex `config_context` used over estimator. Updated by `sklearn_context` if set. |

configs/sklearnex_incremental_example.json

@@ -0,0 +1,65 @@
+{
+    "PARAMETERS_SETS": {
+        "common": {"bench": {"n_runs": 10, "time_limit": 60}},
+        "unlabeled dataset": {
+            "data": [
+                {
+                    "source": "make_blobs",
+                    "generation_kwargs": {
+                        "centers": 1,
+                        "n_samples": 1000,
+                        "n_features": [16, 64]
+                    },
+                    "split_kwargs": {"ignore": true}
+                }
+            ]
+        },
+        "labeled dataset": {
+            "data": {
+                "source": "make_regression",
+                "split_kwargs": {"train_size": 0.2, "test_size": 0.8},
+                "generation_kwargs": {
+                    "n_samples": 5000,
+                    "n_features": [40, 100],
+                    "n_informative": 5,
+                    "noise": 2.0
+                }
+            }
+        },
+        "covariance": {
+            "algorithm": [
+                {
+                    "estimator": "IncrementalEmpiricalCovariance",
+                    "library": "sklearnex.covariance",
+                    "estimator_methods": {"training": "partial_fit"},
+                    "num_batches": {"training": 2}
+                }
+            ]
+        },
+        "linear_regression": {
+            "algorithm": [
+                {
+                    "estimator": "IncrementalLinearRegression",
+                    "library": "sklearnex",
+                    "num_batches": {"training": 2}
+                }
+            ]
+        },
+        "pca": {
+            "algorithm": [
+                {
+                    "estimator": "IncrementalPCA",
+                    "library": "sklearnex.preview",
+                    "num_batches": {"training": 2}
+                }
+            ]
+        }
+    },
+    "TEMPLATES": {
+        "covariance": {"SETS": ["common", "covariance", "unlabeled dataset"]},
+        "linear_regression": {
+            "SETS": ["common", "linear_regression", "labeled dataset"]
+        },
+        "pca": {"SETS": ["common", "pca", "unlabeled dataset"]}
+    }
+}

sklbench/benchmarks/sklearn_estimator.py

@@ -74,7 +74,7 @@ def get_estimator(library_name: str, estimator_name: str):
 def get_estimator_methods(bench_case: BenchCase) -> Dict[str, List[str]]:
     # default estimator methods
     estimator_methods = {
-        "training": ["fit"],
+        "training": ["partial_fit", "fit"],
         "inference": ["predict", "predict_proba", "transform"],
     }
     for stage in estimator_methods.keys():
@@ -334,34 +334,35 @@ def verify_patching(stream: io.StringIO, function_name) -> bool:
     return acceleration_lines > 0 and fallback_lines == 0
 
 
-def create_online_function(method_instance, data_args, batch_size):
-    n_batches = data_args[0].shape[0] // batch_size
+def create_online_function(estimator_instance, method_instance, data_args, num_batches):
 
     if "y" in list(inspect.signature(method_instance).parameters):
 
         def ndarray_function(x, y):
-            for i in range(n_batches):
-                method_instance(
-                    x[i * batch_size : (i + 1) * batch_size],
-                    y[i * batch_size : (i + 1) * batch_size],
-                )
+            for i in range(num_batches):
+                method_instance(x, y)
+            if hasattr(estimator_instance, "_onedal_finalize_fit"):
+                estimator_instance._onedal_finalize_fit()
 
         def dataframe_function(x, y):
-            for i in range(n_batches):
-                method_instance(
-                    x.iloc[i * batch_size : (i + 1) * batch_size],
-                    y.iloc[i * batch_size : (i + 1) * batch_size],
-                )
+            for i in range(num_batches):
+                method_instance(x, y)
+            if hasattr(estimator_instance, "_onedal_finalize_fit"):
+                estimator_instance._onedal_finalize_fit()
 
     else:
 
         def ndarray_function(x):
-            for i in range(n_batches):
-                method_instance(x[i * batch_size : (i + 1) * batch_size])
+            for i in range(num_batches):
+                method_instance(x)
+            if hasattr(estimator_instance, "_onedal_finalize_fit"):
+                estimator_instance._onedal_finalize_fit()
 
         def dataframe_function(x):
-            for i in range(n_batches):
-                method_instance(x.iloc[i * batch_size : (i + 1) * batch_size])
+            for i in range(num_batches):
+                method_instance(x)
+            if hasattr(estimator_instance, "_onedal_finalize_fit"):
+                estimator_instance._onedal_finalize_fit()
 
     if "ndarray" in str(type(data_args[0])):
         return ndarray_function
@@ -417,9 +418,17 @@ def measure_sklearn_estimator(
                 batch_size = get_bench_case_value(
                     bench_case, f"algorithm:batch_size:{stage}"
                 )
-                if batch_size is not None:
+
+                if method == "partial_fit":
+                    num_batches = get_bench_case_value(
+                        bench_case, f"algorithm:num_batches:{stage}", 5
+                    )
+
                     method_instance = create_online_function(
-                        method_instance, data_args, batch_size
+                        estimator_instance,
+                        method_instance,
+                        data_args,
+                        num_batches
                     )
                 # daal4py model builders enabling branch
                 if enable_modelbuilders and stage == "inference":
@@ -536,9 +545,16 @@ def main(bench_case: BenchCase, filters: List[BenchCase]):
     for stage in estimator_methods.keys():
         data_descs[stage].update(
             {
-                "batch_size": get_bench_case_value(
-                    bench_case, f"algorithm:batch_size:{stage}"
-                )
+                key: val
+                for key, val in {
+                    "batch_size": get_bench_case_value(
+                        bench_case, f"algorithm:batch_size:{stage}"
+                    ),
+                    "num_batches": get_bench_case_value(
+                        bench_case, f"algorithm:num_batches:{stage}"
+                    )
+                }.items()
+                if val is not None
             }
         )
         if "n_classes" in data_description:

sklbench/report/implementation.py

@@ -16,7 +16,7 @@
 
 import argparse
 import json
-from typing import Dict, List
+from typing import Dict, Hashable, List
 
 import openpyxl as xl
 import pandas as pd
@@ -94,6 +94,7 @@
     "order",
     "n_classes",
     "n_clusters",
+    "num_batches",
     "batch_size",
 ]
 
@@ -239,6 +240,7 @@ def get_result_tables_as_df(
     bench_cases = pd.DataFrame(
         [flatten_dict(bench_case) for bench_case in results["bench_cases"]]
     )
+    bench_cases = bench_cases.map(lambda x: str(x) if not isinstance(x, Hashable) else x)
 
     if compatibility_mode:
         bench_cases = transform_results_to_compatible(bench_cases)
@@ -248,7 +250,7 @@ def get_result_tables_as_df(
             bench_cases.drop(columns=[column], inplace=True)
             diffby_columns.remove(column)
 
-    return split_df_by_columns(bench_cases, splitby_columns)
+    return split_df_by_columns(bench_cases, splitby_columns, False)
 
 
 def get_summary_from_df(df: pd.DataFrame, df_name: str) -> pd.DataFrame:
@@ -258,7 +260,10 @@ def get_summary_from_df(df: pd.DataFrame, df_name: str) -> pd.DataFrame:
             # only relative improvements are included in summary currently
             if len(column) > 1 and column[1] == f"{metric_name} relative improvement":
                 metric_columns.append(column)
-    summary = df[metric_columns].aggregate(geomean_wrapper, axis=0).to_frame().T
+    if metric_columns:
+        summary = df[metric_columns].aggregate(geomean_wrapper, axis=0).to_frame().T
+    else:
+        summary = pd.DataFrame()
     summary.index = pd.Index([df_name])
     return summary
 

test-configuration-linux.yml

@@ -45,6 +45,11 @@ steps:
       conda activate bench-env
       python -m sklbench --report -l DEBUG --report -c configs/sklearn_example.json
     displayName: Sklearn example run
+  - script: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate bench-env
+      python -m sklbench --report -l DEBUG --report -c configs/sklearnex_incremental_example.json
+    displayName: Incremental algorithms example run
   - script: |
       source /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate bench-env

test-configuration-win.yml

@@ -43,6 +43,10 @@ steps:
       call activate bench-env
       python -m sklbench --report -l DEBUG --report -c configs/sklearn_example.json
     displayName: Sklearn example run
+  - script: |
+      call activate bench-env
+      python -m sklbench --report -l DEBUG --report -c configs/incremental.json
+    displayName: Incremental algorithms example run
   - script: |
       call activate bench-env
       python -m sklbench --report -l DEBUG --report -c configs/xgboost_example.json