fix: transform/convert dead ends and JSON round-trip (#537)

Fixes #537.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: Shashankss1205

src/sktime_mcp/data/adapters/file_adapter.py

@@ -71,9 +71,11 @@ def load(self) -> pd.DataFrame:
             df = self._load_excel(path)
         elif file_format == "parquet":
             df = self._load_parquet(path)
+        elif file_format == "json":
+            df = self._load_json(path)
         else:
             raise ValueError(
-                f"Unsupported format: {file_format}. Supported formats: csv, excel, parquet"
+                f"Unsupported format: {file_format}. Supported formats: csv, excel, parquet, json"
             )
 
         # Set time index
@@ -145,6 +147,7 @@ def _detect_format(self, path: Path) -> str:
             ".xls": "excel",
             ".parquet": "parquet",
             ".pq": "parquet",
+            ".json": "json",
         }
 
         file_format = format_map.get(suffix)
@@ -217,6 +220,16 @@ def _load_parquet(self, path: Path) -> pd.DataFrame:
 
         return df
 
+    def _load_json(self, path: Path) -> pd.DataFrame:
+        """Load a JSON file written by save_data (records orient)."""
+        json_options = self.config.get("json_options", {})
+        json_options.setdefault("orient", "records")
+        try:
+            df = pd.read_json(path, **json_options)
+        except Exception as e:
+            raise ValueError(f"Error reading JSON file: {e}") from e
+        return df
+
     def validate(self, data: pd.DataFrame) -> tuple[bool, dict[str, Any]]:
         """Validate file data using pandas adapter validation."""
         from .pandas_adapter import PandasAdapter

src/sktime_mcp/data/adapters/pandas_adapter.py

@@ -173,10 +173,15 @@ def validate(self, data: pd.DataFrame) -> tuple[bool, dict[str, Any]]:
             missing_pct = (missing_counts / len(data) * 100).round(2)
             warnings.append(f"Missing values detected: {missing_pct[missing_pct > 0].to_dict()}")
 
-        # Check for duplicate indices
+        # Check for duplicate indices — a warning, not a hard error, so the
+        # auto-format step (remove_duplicates) can actually run on the handle.
+        # Rejecting here made that documented remedy unreachable (BUG-19).
         if not isinstance(data.index, pd.MultiIndex) and data.index.duplicated().any():
-            dup_count = data.index.duplicated().sum()
-            errors.append(f"Duplicate time indices found: {dup_count} duplicates")
+            dup_count = int(data.index.duplicated().sum())
+            warnings.append(
+                f"Duplicate time indices found: {dup_count}. They will be de-duplicated "
+                "by auto-format (keeping the first of each)."
+            )
 
         # Check for monotonic index
         if not data.index.is_monotonic_increasing:

src/sktime_mcp/tools/save_data.py

@@ -110,14 +110,18 @@ def save_data_tool(
         abs_path.parent.mkdir(parents=True, exist_ok=True)
 
         # Write
-        writer = getattr(df, _FORMAT_WRITERS[fmt])
         if fmt == "json":
-            writer(str(abs_path), orient="records", date_format="iso", indent=2)
+            # records orient drops the index; write it as a "time" column so the
+            # file round-trips through load_data_source(time_column="time").
+            out_df = df.copy()
+            out_df.index = out_df.index.astype(str)
+            out_df = out_df.reset_index(names="time")
+            out_df.to_json(str(abs_path), orient="records", indent=2)
         elif fmt == "parquet":
-            writer(str(abs_path))
+            df.to_parquet(str(abs_path))
         else:
             # CSV — include the index as a time column
-            writer(str(abs_path))
+            df.to_csv(str(abs_path))
 
         return {
             "success": True,

src/sktime_mcp/tools/transform_data.py

@@ -159,6 +159,11 @@ def _action_format(
 # ---------------------------------------------------------------------------
 
 
+# Index-less mtypes: converting a handle to these strips the time index, which
+# breaks every downstream tool (inspect/split/format) and fabricates a cutoff.
+_INDEXLESS_MTYPES = {"np.ndarray", "numpy3D", "numpyflat", "numpy2D"}
+
+
 def _action_convert(
     executor: Any,
     data_handle: str,
@@ -170,8 +175,33 @@ def _action_convert(
 
     from sktime.datatypes import convert_to
 
+    if to_mtype in _INDEXLESS_MTYPES:
+        return {
+            "success": False,
+            "error": (
+                f"'{to_mtype}' has no time index; converting a handle to it breaks "
+                "inspect_data/split_data and other tools. Convert to a pandas mtype "
+                "(pd.Series, pd.DataFrame) instead."
+            ),
+        }
+
     original_mtype = type(y).__name__
-    converted = convert_to(y, to_type=to_mtype)
+    try:
+        converted = convert_to(y, to_type=to_mtype)
+    except (TypeError, ValueError, KeyError) as e:
+        msg = str(e)
+        # sktime raises a multi-paragraph mtype-inference dump for a
+        # scitype-incompatible target (e.g. Series handle -> pd-multiindex).
+        if "No valid mtype" in msg or "must be of python type" in msg:
+            return {
+                "success": False,
+                "error": (
+                    f"Cannot convert a {original_mtype} (Series-scitype) handle to "
+                    f"'{to_mtype}'. That target expects a different scitype (e.g. Panel). "
+                    "Use a compatible mtype such as pd.Series or pd.DataFrame."
+                ),
+            }
+        return {"success": False, "error": f"Conversion to '{to_mtype}' failed: {msg}"}
 
     # Register as new handle
     new_handle = f"data_{uuid.uuid4().hex[:8]}"

tests/test_transform_deadends.py

@@ -0,0 +1,94 @@
+"""transform/convert dead ends and JSON round-trip (#537)."""
+
+import contextlib
+import os
+import tempfile
+
+import pandas as pd
+import pytest
+
+from sktime_mcp.runtime.executor import get_executor
+from sktime_mcp.tools.save_data import save_data_tool
+from sktime_mcp.tools.transform_data import transform_data_tool
+
+
+@pytest.fixture
+def series_handle():
+    ex = get_executor()
+    res = ex.load_data_source(
+        {
+            "type": "pandas",
+            "data": {
+                "date": [f"2024-{m:02d}-01" for m in range(1, 13)],
+                "value": [float(i) for i in range(12)],
+            },
+            "time_column": "date",
+            "target_column": "value",
+        }
+    )
+    dh = res["data_handle"]
+    yield ex, dh
+    for h in list(ex._data_handles):
+        if h == dh:
+            ex._data_handles.pop(h, None)
+
+
+class TestConvertDeadEnds:
+    def test_ndarray_target_rejected(self, series_handle):
+        ex, dh = series_handle
+        res = transform_data_tool(data_handle=dh, action="convert", to_mtype="np.ndarray")
+        assert not res["success"]
+        assert "no time index" in res["error"].lower()
+
+    def test_series_to_panel_clean_error(self, series_handle):
+        ex, dh = series_handle
+        res = transform_data_tool(data_handle=dh, action="convert", to_mtype="pd-multiindex")
+        assert not res["success"]
+        # clean domain message, not a multi-paragraph mtype dump
+        assert "scitype" in res["error"].lower() or "panel" in res["error"].lower()
+        assert "No valid mtype" not in res["error"]
+
+    def test_valid_convert_still_works(self, series_handle):
+        ex, dh = series_handle
+        res = transform_data_tool(data_handle=dh, action="convert", to_mtype="pd.DataFrame")
+        assert res["success"], res
+
+
+class TestDuplicateTimestamps:
+    def test_duplicates_load_and_dedup(self):
+        ex = get_executor()
+        res = ex.load_data_source(
+            {
+                "type": "pandas",
+                "data": {
+                    "date": ["2024-01-01", "2024-01-01", "2024-02-01", "2024-03-01"],
+                    "value": [1.0, 1.5, 2.0, 3.0],
+                },
+                "time_column": "date",
+                "target_column": "value",
+            }
+        )
+        try:
+            # previously this hard-failed with "Duplicate time indices found"
+            assert res["success"], res
+            # auto-format removed the duplicate
+            handle = res["data_handle"]
+            assert not ex._data_handles[handle]["y"].index.duplicated().any()
+        finally:
+            ex._data_handles.pop(res.get("data_handle"), None)
+
+
+class TestJsonRoundTrip:
+    def test_json_save_then_load(self, series_handle):
+        ex, dh = series_handle
+        with tempfile.TemporaryDirectory() as d:
+            path = os.path.join(d, "data.json")
+            saved = save_data_tool(dh, path=path, format="json")
+            assert saved["success"], saved
+            loaded = ex.load_data_source(
+                {"type": "file", "path": path, "time_column": "time", "target_column": "value"}
+            )
+            try:
+                assert loaded["success"], loaded
+            finally:
+                ex._data_handles.pop(loaded.get("data_handle"), None)