Preserve no-default semantics in CTable schemas

   Omit "default" during schema serialization when no default is declared,
   so explicit default=None can round-trip distinctly from dataclasses.MISSING.

   Also improve CTable append/extend behavior and errors for omitted columns:
   columns with declared defaults are filled automatically, while columns with
   no default declared produce clearer ValueError messages.

Author: FrancescAlted

src/blosc2/ctable.py

@@ -3735,16 +3735,16 @@ def add_column(
         spec:
             A schema descriptor such as ``b2.int64(ge=0)`` or a field
             descriptor such as ``b2.field(b2.int64(ge=0), default=0)``.
-            A default is required when the table already has live rows, so
-            those rows can be backfilled.
+            When the table already has live rows, use ``blosc2.field(...)``
+            with a default declared so those rows can be backfilled.
 
         Raises
         ------
         ValueError
             If the table is read-only, is a view, the column already exists,
-            or a non-empty table is given a column without a default value.
+            or a non-empty table is given a column with no default declared.
         TypeError
-            If *default* cannot be coerced to *spec*'s dtype.
+            If a declared default cannot be coerced to *spec*'s dtype.
         """
         if self._read_only:
             raise ValueError("Table is read-only (opened with mode='r').")
@@ -4111,6 +4111,28 @@ def _autofill_materialized_row_values(self, row: dict[str, Any]) -> dict[str, An
             row[name] = np.asarray(values, dtype=meta["dtype"])[0]
         return row
 
+    def _validate_no_default_columns_present(self, row: dict[str, Any]) -> None:
+        """Raise a clear error when a row omits a column with no default declared."""
+        for col in self._schema.columns:
+            if col.name in row:
+                continue
+            is_nullable = getattr(col.spec, "null_value", None) is not None or bool(
+                getattr(col.spec, "nullable", False)
+            )
+            if col.default is MISSING and not is_nullable:
+                raise ValueError(f"Column {col.name!r} has no default declared; a value must be provided.")
+
+    def _fill_default_batch_columns(self, raw_columns: dict[str, Any], row_count: int) -> dict[str, Any]:
+        """Fill omitted batch columns from defaults, or raise if no default is declared."""
+        raw_columns = dict(raw_columns)
+        for col in self._schema.columns:
+            if col.name in raw_columns:
+                continue
+            if col.default is MISSING:
+                raise ValueError(f"Column {col.name!r} has no default declared; values must be provided.")
+            raw_columns[col.name] = [col.default] * row_count
+        return raw_columns
+
     def _autofill_materialized_batch_columns(
         self, raw_columns: dict[str, Any], row_count: int, *, provided_names: set[str]
     ) -> dict[str, Any]:
@@ -5974,6 +5996,7 @@ def append(self, data: list | np.void | np.ndarray) -> None:
         # Normalize → validate → coerce
         row = self._normalize_row_input(data)
         row = self._autofill_materialized_row_values(row)
+        self._validate_no_default_columns_present(row)
         if self._validate:
             from blosc2.schema_validation import validate_row
 
@@ -6033,7 +6056,8 @@ def extend(self, data: list | CTable | Any, *, validate: bool | None = None) ->
         *data* may be:
 
         * a **dict of arrays** ``{"col": array, ...}`` — all arrays must have
-          the same length; missing columns are filled with their default value;
+          the same length; omitted columns are filled from their declared default;
+          columns with no default declared must be provided;
         * a **list of rows**, each compatible with :meth:`append`;
         * another **CTable** — columns are matched by name.
 
@@ -6103,6 +6127,7 @@ def extend(self, data: list | CTable | Any, *, validate: bool | None = None) ->
         raw_columns = self._autofill_materialized_batch_columns(
             raw_columns, new_nrows, provided_names=provided_names
         )
+        raw_columns = self._fill_default_batch_columns(raw_columns, new_nrows)
 
         # Validate constraints column-by-column before writing
         if do_validate:

src/blosc2/schema_compiler.py

@@ -149,7 +149,7 @@ class CompiledColumn:
     py_type: Any
     spec: SchemaSpec
     dtype: np.dtype | None
-    default: Any  # MISSING means required (no default)
+    default: Any  # MISSING means no default declared
     config: ColumnConfig
     display_width: int = 20  # terminal column width for __str__ / info()
 
@@ -360,9 +360,9 @@ def _json_to_bytes(value: dict[str, Any]) -> bytes:
 
 
 def _default_to_json(value: Any) -> Any:
-    """Convert a field default to a JSON-compatible value."""
+    """Convert a declared field default to a JSON-compatible value."""
     if value is MISSING:
-        return None
+        raise ValueError("Cannot serialize MISSING as a declared default.")
     if isinstance(value, complex):
         return {"__complex__": True, "real": value.real, "imag": value.imag}
     if isinstance(value, bytes):
@@ -372,8 +372,6 @@ def _default_to_json(value: Any) -> Any:
 
 def _default_from_json(value: Any) -> Any:
     """Reverse of :func:`_default_to_json`."""
-    if value is None:
-        return MISSING
     if isinstance(value, dict) and value.get("__complex__"):
         return complex(value["real"], value["imag"])
     if isinstance(value, dict) and value.get("__bytes__"):
@@ -411,7 +409,7 @@ def schema_to_dict(schema: CompiledSchema) -> dict[str, Any]:
             "version": 1,
             "row_cls": "Row",
             "columns": [
-                {"name": "id", "kind": "int64", "ge": 0, "default": null},
+                {"name": "id", "kind": "int64", "ge": 0},
                 {"name": "score", "kind": "float64", "ge": 0, "le": 100, "default": 0.0},
                 {"name": "active", "kind": "bool", "default": true},
             ]
@@ -423,7 +421,8 @@ def schema_to_dict(schema: CompiledSchema) -> dict[str, Any]:
         entry.update(col.spec.to_metadata_dict())  # adds "kind" + constraints
         if isinstance(entry.get("null_value"), bytes):
             entry["null_value"] = _bytes_to_json(entry["null_value"])
-        entry["default"] = _default_to_json(col.default)
+        if col.default is not MISSING:
+            entry["default"] = _default_to_json(col.default)
         if col.config.cparams is not None:
             entry["cparams"] = col.config.cparams
         if col.config.dparams is not None:
@@ -465,7 +464,7 @@ def schema_from_dict(data: dict[str, Any]) -> CompiledSchema:
         entry = dict(entry)  # don't mutate caller's data
         name = entry.pop("name")
         kind = entry.pop("kind")
-        default = _default_from_json(entry.pop("default", None))
+        default = _default_from_json(entry.pop("default")) if "default" in entry else MISSING
         cparams = entry.pop("cparams", None)
         dparams = entry.pop("dparams", None)
         chunks = tuple(entry.pop("chunks")) if "chunks" in entry else None

tests/ctable/test_schema_compiler.py

@@ -46,6 +46,12 @@ class WithComplex:
     c_val: complex = blosc2.field(blosc2.complex128(), default=0j)
 
 
+@dataclass
+class WithNoneDefault:
+    id: int = blosc2.field(blosc2.int64())
+    note: str = blosc2.field(blosc2.vlstring(nullable=True), default=None)
+
+
 # -------------------------------------------------------------------
 # compile_schema — explicit b2.field()
 # -------------------------------------------------------------------
@@ -84,7 +90,7 @@ def test_compile_column_specs():
 
 def test_compile_defaults():
     s = compile_schema(Simple)
-    assert s.columns_by_name["id"].default is MISSING  # required
+    assert s.columns_by_name["id"].default is MISSING  # no default declared
     assert s.columns_by_name["score"].default == 0.0
     assert s.columns_by_name["active"].default is True
 
@@ -193,7 +199,7 @@ def test_schema_to_dict_column_fields():
     id_col = next(c for c in d["columns"] if c["name"] == "id")
     assert id_col["kind"] == "int64"
     assert id_col["ge"] == 0
-    assert id_col["default"] is None  # MISSING → None
+    assert "default" not in id_col  # no default declared omits the key
 
 
 def test_schema_to_dict_default_values():
@@ -213,6 +219,20 @@ def test_schema_to_dict_complex_default():
     assert c_col["default"]["imag"] == 0.0
 
 
+def test_schema_to_dict_none_default():
+    d = schema_to_dict(compile_schema(WithNoneDefault))
+    id_col = next(c for c in d["columns"] if c["name"] == "id")
+    note_col = next(c for c in d["columns"] if c["name"] == "note")
+    assert "default" not in id_col
+    assert note_col["default"] is None
+
+
+def test_schema_roundtrip_none_default():
+    restored = schema_from_dict(schema_to_dict(compile_schema(WithNoneDefault)))
+    assert restored.columns_by_name["id"].default is MISSING
+    assert restored.columns_by_name["note"].default is None
+
+
 def test_schema_roundtrip():
     """schema_from_dict(schema_to_dict(s)) reproduces the same column structure."""
     original = compile_schema(Simple)

tests/ctable/test_schema_validation.py

@@ -60,12 +60,18 @@ def test_append_boundary_values():
 def test_append_default_fill():
     """Fields with defaults can be omitted from a tuple — Pydantic fills them in."""
     t = CTable(Row, expected_size=5)
-    # Only id is required; score and active have defaults
+    # Only id has no default declared; score and active have defaults
     t.append((5,))  # score=0.0, active=True filled by defaults
     assert len(t) == 1
     assert t[0].id == 5
 
 
+def test_append_omitted_no_default_column_raises_clear_error():
+    t = CTable(Row, expected_size=5)
+    with pytest.raises(ValueError, match="no default declared"):
+        t.append(())
+
+
 def test_append_validate_false():
     """validate=False skips constraint checks — invalid data is stored silently."""
     t = CTable(Row, expected_size=5, validate=False)
@@ -103,6 +109,19 @@ def test_extend_le_violation():
         t.extend(data)
 
 
+def test_extend_omitted_columns_with_defaults_are_filled():
+    t = CTable(Row, expected_size=10)
+    t.extend({"id": [1, 2]})
+    assert list(t["score"][:]) == [0.0, 0.0]
+    assert list(t["active"][:]) == [True, True]
+
+
+def test_extend_omitted_no_default_column_raises_clear_error():
+    t = CTable(Row, expected_size=10)
+    with pytest.raises(ValueError, match="no default declared"):
+        t.extend({"score": [1.0, 2.0]})
+
+
 def test_extend_validate_false():
     """validate=False on the table skips bulk constraint checks."""
     t = CTable(Row, expected_size=10, validate=False)