rest_api: Unify static and dynamic attributes handling

Replace the Multinested-dependent schemes in host config with the new
static + dynamic schemes.

CMK-22453

Change-Id: If84a80b08a76cc3c8322ea95c9fff9389dec8eac

Author: lpetrora

cmk/gui/fields/definitions.py

@@ -12,10 +12,11 @@
 import typing
 import uuid
 import warnings
-from collections.abc import Callable, Collection, Mapping
+from collections.abc import Callable, Collection, Mapping, MutableMapping
 from datetime import datetime, timezone
 from typing import Any, Literal
 
+import marshmallow
 from cryptography.x509 import CertificateSigningRequest, load_pem_x509_csr
 from cryptography.x509.oid import NameOID
 from marshmallow import fields as _fields
@@ -32,11 +33,11 @@
 from cmk.utils.livestatus_helpers.tables import Hostgroups, Hosts, Servicegroups
 from cmk.utils.livestatus_helpers.types import Column, Table
 from cmk.utils.regex import regex, REGEX_ID
-from cmk.utils.tags import TagGroupID, TagID
+from cmk.utils.tags import TagConfig, TagGroup, TagGroupID, TagID
 from cmk.utils.user import UserId
 
 from cmk.gui import sites
-from cmk.gui.config import builtin_role_ids
+from cmk.gui.config import active_config, builtin_role_ids
 from cmk.gui.customer import customer_api, SCOPE_GLOBAL
 from cmk.gui.exceptions import MKUserError
 from cmk.gui.fields.base import BaseSchema, MultiNested, ValueTypedDictSchema
@@ -48,11 +49,11 @@
 from cmk.gui.userdb import load_users
 from cmk.gui.watolib import userroles
 from cmk.gui.watolib.groups_io import load_group_information
-from cmk.gui.watolib.host_attributes import host_attribute
+from cmk.gui.watolib.host_attributes import ABCHostAttribute, all_host_attributes, host_attribute
 from cmk.gui.watolib.hosts_and_folders import Folder, folder_tree, Host
 from cmk.gui.watolib.passwords import contact_group_choices, password_exists
 from cmk.gui.watolib.sites import site_management_registry
-from cmk.gui.watolib.tags import load_tag_group
+from cmk.gui.watolib.tags import load_tag_config_read_only, load_tag_group
 
 from cmk.fields import base, Boolean, DateTime, validators
 
@@ -734,6 +735,90 @@ def validate_custom_host_attributes(
     return host_attributes
 
 
+class CustomHostAttributesAndTagGroups(BaseSchema):
+    class Meta:
+        unknown = marshmallow.INCLUDE
+
+    # Set it to true on create and update schemas to raise an error if a readonly attribute is passed
+    _raise_error_if_attribute_is_readonly = False
+
+    @marshmallow.post_load(pass_original=True)
+    def _validate_extra_attributes(
+        self,
+        result_data: dict[str, Any],
+        original_data: MutableMapping[str, Any],
+        **_unused_args: Any,
+    ) -> dict[str, Any]:
+        for field in self.fields:
+            original_data.pop(field, None)
+
+        if not original_data:
+            return result_data
+
+        host_attributes = all_host_attributes(active_config)
+        tag_group_config = load_tag_config_read_only()
+
+        for name, value in original_data.items():
+            if tag_group := self._get_custom_tag_group(name, tag_group_config):
+                self._validate_tag_group(tag_group, value)
+
+            elif host_attribute := self._get_custom_host_attribute(name, host_attributes):
+                self._validate_attribute(host_attribute, value)
+
+            else:
+                self._raise_error(f"Unknown Attribute: {name!r}: {value!r}")
+
+            result_data[name] = value
+        return result_data
+
+    @marshmallow.post_dump(pass_original=True)
+    def _add_tags_and_custom_attributes_back(
+        self, dump_data: dict[str, Any], original_data: dict[str, Any], **_kwargs: Any
+    ) -> dict[str, Any]:
+        # Custom attributes and tags are thrown away during validation as they have no field in the schema.
+        # So we dump them back in here.
+        # TODO: This code complies with the behavior enforced by the test_openapi_host_has_deleted_custom_attributes
+        #       test. However more research is needed to determine if it should change.
+        original_data.update(dump_data)
+        return original_data
+
+    def _get_custom_tag_group(self, tag_name: str, tag_config: TagConfig) -> TagGroup | None:
+        return tag_config.get_tag_group(TagGroupID(tag_name[4:]))
+
+    def _get_custom_host_attribute(
+        self, attribute_name: str, attributes: dict[str, ABCHostAttribute]
+    ) -> ABCHostAttribute | None:
+        try:
+            attribute = attributes[attribute_name]
+            if not attribute.from_config():
+                return None
+
+            return attribute
+
+        except KeyError:
+            return None
+
+    def _validate_attribute(self, host_attribute: ABCHostAttribute, value: object) -> None:
+        if self._raise_error_if_attribute_is_readonly and not host_attribute.editable():
+            self._raise_error(f"Attribute {host_attribute.name()!r} is readonly.")
+
+        if not isinstance(value, str):
+            self._raise_error(f"Attribute {host_attribute.name()!r} must be a string.")
+
+        try:
+            host_attribute.validate_input(value, "")
+
+        except MKUserError as exc:
+            self._raise_error(f"{host_attribute.name()}: {str(exc)}")
+
+    def _validate_tag_group(self, tag_group: TagGroup, value: object) -> None:
+        if value not in tag_group.get_tag_ids():
+            self._raise_error(f"Invalid value for tag-group {tag_group.title!r}: {value!r}")
+
+    def _raise_error(self, message: str) -> None:
+        raise ValidationError(message)
+
+
 def ensure_string(value):
     if not isinstance(value, str):
         raise ValidationError(f"Not a string, but a {type(value).__name__}")
@@ -1067,7 +1152,7 @@ def _validate(self, value):
         if (
             self.presence == "might_not_exist_on_view"
             and self.context is not None
-            and self.context["object_context"] == "view"
+            and self.context.get("object_context") == "view"
         ):
             return
 

cmk/gui/openapi/endpoints/_common/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python3
+# Copyright (C) 2025 Checkmk GmbH - License: GNU General Public License v2
+# This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
+# conditions defined in the file COPYING, which is part of this source code package.

cmk/gui/openapi/endpoints/_common/folder_attribute_schemas.py

@@ -0,0 +1,86 @@
+# !/usr/bin/env python3
+# Copyright (C) 2025 Checkmk GmbH - License: GNU General Public License v2
+# This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
+# conditions defined in the file COPYING, which is part of this source code package.
+
+
+from cmk.gui import fields as gui_fields
+from cmk.gui.fields.attributes import HostContactGroup
+from cmk.gui.fields.base import BaseSchema
+from cmk.gui.fields.definitions import CustomHostAttributesAndTagGroups
+from cmk.gui.watolib.builtin_attributes import (
+    HostAttributeLabels,
+    HostAttributeManagementIPMICredentials,
+    HostAttributeManagementProtocol,
+    HostAttributeManagementSNMPCommunity,
+    HostAttributeMetaData,
+    HostAttributeNetworkScan,
+    HostAttributeNetworkScanResult,
+    HostAttributeParents,
+    HostAttributeSite,
+    HostAttributeSNMPCommunity,
+)
+from cmk.gui.watolib.groups import HostAttributeContactGroups
+from cmk.gui.watolib.host_attributes import ABCHostAttribute
+
+from cmk import fields
+
+from .host_attribute_schemas import BaseHostTagGroup
+
+
+class BaseFolderTagGroup(BaseHostTagGroup):
+    pass
+
+
+class BaseFolderAttribute(BaseSchema):
+    """Base class for all folder attribute schemas."""
+
+    site = HostAttributeSite().openapi_field()
+    parents = HostAttributeParents().openapi_field()
+
+    contactgroups = fields.Nested(
+        HostContactGroup,
+        description=HostAttributeContactGroups().help(),
+    )
+
+    bake_agent_package = gui_fields.bake_agent_field()
+    snmp_community = HostAttributeSNMPCommunity().openapi_field()
+
+    labels = HostAttributeLabels().openapi_field()
+
+    network_scan = HostAttributeNetworkScan().openapi_field()
+
+    management_protocol = HostAttributeManagementProtocol().openapi_field()
+    management_snmp_community = HostAttributeManagementSNMPCommunity().openapi_field()
+    management_ipmi_credentials = HostAttributeManagementIPMICredentials().openapi_field()
+
+
+class FolderCustomHostAttributesAndTagGroups(CustomHostAttributesAndTagGroups):
+    def _get_custom_host_attribute(
+        self, attribute_name: str, attributes: dict[str, ABCHostAttribute]
+    ) -> ABCHostAttribute | None:
+        attribute = super()._get_custom_host_attribute(attribute_name, attributes)
+
+        if attribute and not attribute.show_in_folder():
+            attribute = None
+
+        return attribute
+
+
+class FolderCreateAttribute(
+    BaseFolderAttribute, BaseFolderTagGroup, FolderCustomHostAttributesAndTagGroups
+):
+    _raise_error_if_attribute_is_readonly = True
+
+
+class FolderUpdateAttribute(
+    BaseFolderAttribute, BaseFolderTagGroup, FolderCustomHostAttributesAndTagGroups
+):
+    _raise_error_if_attribute_is_readonly = True
+
+
+class FolderViewAttribute(
+    BaseFolderAttribute, BaseFolderTagGroup, FolderCustomHostAttributesAndTagGroups
+):
+    network_scan_result = HostAttributeNetworkScanResult().openapi_field()
+    meta_data = HostAttributeMetaData().openapi_field()

cmk/gui/openapi/endpoints/host_config/attribute_schemas.py cmk/gui/openapi/endpoints/_common/host_attribute_schemas.py

@@ -2,9 +2,12 @@
 # Copyright (C) 2025 Checkmk GmbH - License: GNU General Public License v2
 # This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
 # conditions defined in the file COPYING, which is part of this source code package.
+from cmk.utils.tags import BuiltinTagConfig, TagGroupID
+
 from cmk.gui import fields as gui_fields
 from cmk.gui.fields.attributes import HostContactGroup
-from cmk.gui.fields.utils import BaseSchema
+from cmk.gui.fields.base import BaseSchema
+from cmk.gui.fields.definitions import CustomHostAttributesAndTagGroups
 from cmk.gui.watolib.builtin_attributes import (
     HostAttributeAdditionalIPv4Addresses,
     HostAttributeAdditionalIPv6Addresses,
@@ -31,6 +34,41 @@
 
 from cmk import fields
 
+built_in_tag_group_config = BuiltinTagConfig()
+
+
+def _get_valid_tag_ids(built_in_tag_group_id: TagGroupID) -> list[str | None]:
+    tag_group = built_in_tag_group_config.get_tag_group(built_in_tag_group_id)
+    assert tag_group is not None
+
+    return [None if tag_id is None else str(tag_id) for tag_id in tag_group.get_tag_ids()]
+
+
+class BaseHostTagGroup(BaseSchema):
+    tag_address_family = fields.String(
+        description="The IP address family of the host.",
+        example="ip-v4-only",
+        enum=_get_valid_tag_ids(TagGroupID("address_family")),
+    )
+
+    tag_agent = fields.String(
+        description="Agent and API integrations",
+        example="cmk-agent",
+        enum=_get_valid_tag_ids(TagGroupID("agent")),
+    )
+
+    tag_snmp_ds = fields.String(
+        description="The SNMP data source of the host.",
+        example="snmp-v2",
+        enum=_get_valid_tag_ids(TagGroupID("snmp_ds")),
+    )
+
+    tag_piggyback = fields.String(
+        description="Use piggyback data for this host.",
+        example="piggyback",
+        enum=_get_valid_tag_ids(TagGroupID("piggyback")),
+    )
+
 
 class BaseHostAttribute(BaseSchema):
     """Base class for all host attribute schemas."""
@@ -67,18 +105,21 @@ class BaseHostAttribute(BaseSchema):
     inventory_failed = HostAttributeDiscoveryFailed().openapi_field()
 
 
-class HostCreateAttribute(BaseHostAttribute):
-    pass
+class HostCreateAttribute(BaseHostAttribute, BaseHostTagGroup, CustomHostAttributesAndTagGroups):
+    _raise_error_if_attribute_is_readonly = True
+
 
+class HostViewAttribute(BaseHostAttribute, BaseHostTagGroup, CustomHostAttributesAndTagGroups):
+    class Meta:
+        dateformat = "iso8601"
 
-class HostViewAttribute(BaseHostAttribute):
     network_scan_result = HostAttributeNetworkScanResult().openapi_field()
     meta_data = HostAttributeMetaData().openapi_field()
 
 
-class HostUpdateAttribute(BaseHostAttribute):
-    pass
+class HostUpdateAttribute(BaseHostAttribute, BaseHostTagGroup, CustomHostAttributesAndTagGroups):
+    _raise_error_if_attribute_is_readonly = True
 
 
-class ClusterCreateAttribute(BaseHostAttribute):
-    pass
+class ClusterCreateAttribute(BaseHostAttribute, BaseHostTagGroup, CustomHostAttributesAndTagGroups):
+    _raise_error_if_attribute_is_readonly = True

cmk/gui/openapi/endpoints/folder_config/request_schemas.py

@@ -11,6 +11,10 @@
 
 from cmk.gui import fields as gui_fields
 from cmk.gui.fields.utils import BaseSchema
+from cmk.gui.openapi.endpoints._common.folder_attribute_schemas import (
+    FolderCreateAttribute,
+    FolderUpdateAttribute,
+)
 from cmk.gui.openapi.endpoints.common_fields import EXISTING_FOLDER, EXISTING_FOLDER_PATTERN
 
 from cmk import fields
@@ -56,15 +60,15 @@ class CreateFolder(BaseSchema):
         example="/",
         pattern=EXISTING_FOLDER_PATTERN,
     )
-    attributes = gui_fields.host_attributes_field(
-        "folder",
-        "create",
-        "inbound",
+
+    attributes = fields.Nested(
+        FolderCreateAttribute,
         required=False,
         description=(
-            "Specific attributes to apply for all hosts in this folder (among other things)."
+            "Specific attributes to apply for all hosts in this folder (among other things). Built-in and custom attributes and tag groups can be set here."
         ),
         example={"tag_criticality": "prod"},
+        load_default=dict(),
     )
 
 
@@ -92,32 +96,29 @@ class UpdateFolder(BaseSchema):
         required=False,
         description="The title of the folder. Used in the GUI.",
     )
-    attributes = gui_fields.host_attributes_field(
-        "folder",
-        "update",
-        "inbound",
+
+    attributes = fields.Nested(
+        FolderUpdateAttribute,
         description=(
             "Replace all attributes with the ones given in this field. Already set"
             "attributes, not given here, will be removed. Can't be used together with "
             "update_attributes or remove_attributes fields."
         ),
         example={"tag_networking": "wan"},
         required=False,
-        load_default=None,
     )
-    update_attributes = gui_fields.host_attributes_field(
-        "folder",
-        "update",
-        "inbound",
+
+    update_attributes = fields.Nested(
+        FolderUpdateAttribute,
         description=(
             "Just update the folder attributes with these attributes. The previously set "
             "attributes will be overwritten. Can't be used together with attributes or "
             "remove_attributes fields."
         ),
         example={"tag_criticality": "prod"},
         required=False,
-        load_default=None,
     )
+
     remove_attributes = fields.List(
         fields.String(),
         description=(