fix: reject non-forecasters and non-numeric targets in evaluate (#535)

Validate object_type == forecaster and a numeric Series y before CV, syncing
the async path to reject up front. Fixes #535.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: Shashankss1205

src/sktime_mcp/tools/evaluate.py

@@ -14,6 +14,75 @@
 logger = logging.getLogger(__name__)
 
 
+def _validate_evaluate_inputs(executor, estimator_handle: str, y: str) -> dict[str, Any] | None:
+    """Reject non-forecasters and non-Series targets before any CV work.
+
+    Returns an error dict to short-circuit on, or None when inputs are valid.
+    Cross-validation here uses an ExpandingWindowSplitter and forecasting
+    metrics, which only make sense for a forecaster on a single Series.
+    """
+    import pandas as pd
+
+    try:
+        instance = executor._handle_manager.get_instance(estimator_handle)
+    except KeyError:
+        return {"success": False, "error": executor._handle_manager.describe_missing(estimator_handle)}
+
+    get_tag = getattr(instance, "get_class_tag", None)
+    obj_type = get_tag("object_type", "") if callable(get_tag) else ""
+    if obj_type != "forecaster":
+        return {
+            "success": False,
+            "error": (
+                f"evaluate cross-validates forecasters, but this handle is a "
+                f"{obj_type or 'non-forecaster object'}. Use call_method for "
+                "non-forecaster estimators."
+            ),
+        }
+
+    # Resolve y and confirm it is a numeric Series (not Panel/Hierarchical, and
+    # not categorical label data from a classification dataset).
+    y_res = executor._resolve_source(y)
+    if not y_res["success"]:
+        return y_res
+    _y = y_res["data"]
+    try:
+        from sktime.datatypes import check_is_scitype
+
+        is_series = check_is_scitype(_y, scitype="Series", return_metadata=[])[0]
+    except Exception:
+        is_series = True  # let the downstream run surface an unusual case
+    if not is_series:
+        return {
+            "success": False,
+            "error": (
+                "evaluate expects a univariate/Series target, but the given y is Panel "
+                "or Hierarchical scitype. Forecasting CV is not defined for panel data."
+            ),
+        }
+
+    import numpy as np
+
+    if isinstance(_y, pd.Series):
+        numeric = pd.api.types.is_numeric_dtype(_y)
+    elif isinstance(_y, pd.DataFrame):
+        numeric = all(pd.api.types.is_numeric_dtype(_y[c]) for c in _y.columns)
+    elif isinstance(_y, np.ndarray):
+        numeric = np.issubdtype(_y.dtype, np.number)
+    else:
+        numeric = True
+    if not numeric:
+        return {
+            "success": False,
+            "error": (
+                f"evaluate needs a numeric forecasting target, but y '{y}' is non-numeric "
+                "(categorical/label data — this looks like classification data, not a "
+                "forecasting series)."
+            ),
+        }
+    return None
+
+
 def evaluate_tool(
     estimator_handle: str,
     y: str,
@@ -42,6 +111,12 @@ def evaluate_tool(
 
     executor = get_executor()
 
+    # Scitype validation up front, so async calls reject synchronously instead
+    # of burning a background job on invalid inputs (#535).
+    invalid = _validate_evaluate_inputs(executor, estimator_handle, y)
+    if invalid is not None:
+        return invalid
+
     if run_async:
         job_manager = get_job_manager()
         try:

tests/test_evaluate_scitype.py

@@ -0,0 +1,79 @@
+"""evaluate must reject non-forecasters and non-Series targets up front (#535).
+
+Previously it attempted cross-validation on a transformer/int handle or on
+Panel data, wasting time and (before the error_score fix) masking the result.
+"""
+
+import contextlib
+
+import pytest
+
+from sktime_mcp.runtime.executor import get_executor
+from sktime_mcp.runtime.handles import get_handle_manager
+from sktime_mcp.tools.evaluate import evaluate_tool
+from sktime_mcp.tools.instantiate import instantiate_tool
+
+
+def _release(handle):
+    with contextlib.suppress(KeyError):
+        get_handle_manager().release_handle(handle)
+
+
+def test_rejects_transformer():
+    res = instantiate_tool(spec="Deseasonalizer()")
+    handle = res["handle"]
+    try:
+        out = evaluate_tool(estimator_handle=handle, y="airline", cv_folds=3)
+        assert not out["success"]
+        assert "forecaster" in out["error"].lower()
+        assert "transformer" in out["error"].lower()
+    finally:
+        _release(handle)
+
+
+def test_rejects_non_estimator():
+    res = instantiate_tool(spec="42")
+    handle = res["handle"]
+    try:
+        out = evaluate_tool(estimator_handle=handle, y="airline", cv_folds=3)
+        assert not out["success"]
+        # int handle has no forecaster object_type -> rejected (here or by scitype)
+        assert "forecaster" in out["error"].lower() or "series" in out["error"].lower()
+    finally:
+        _release(handle)
+
+
+def test_rejects_classification_dataset_target():
+    """basic_motions resolves to categorical labels — not a forecasting series."""
+    res = instantiate_tool(spec="NaiveForecaster()")
+    handle = res["handle"]
+    try:
+        out = evaluate_tool(estimator_handle=handle, y="basic_motions", cv_folds=3)
+        assert not out["success"]
+        err = out["error"].lower()
+        assert "numeric" in err or "panel" in err or "series" in err or "classification" in err
+    finally:
+        _release(handle)
+
+
+def test_forecaster_on_series_still_works():
+    res = instantiate_tool(spec="NaiveForecaster(sp=12)")
+    handle = res["handle"]
+    try:
+        out = evaluate_tool(estimator_handle=handle, y="airline", cv_folds=3)
+        assert out["success"], out
+        assert out["metrics"]
+    finally:
+        _release(handle)
+
+
+def test_async_rejects_synchronously():
+    """Invalid inputs must not burn a background job."""
+    res = instantiate_tool(spec="Deseasonalizer()")
+    handle = res["handle"]
+    try:
+        out = evaluate_tool(estimator_handle=handle, y="airline", cv_folds=3, run_async=True)
+        assert not out["success"]
+        assert "job_id" not in out
+    finally:
+        _release(handle)