agent_based: Refactor get_relevant_raw_sections

This refactors (renames, and moves) the function
to avoid accessing the global state by taking
the plugins themselves (instead of their names)
as arguments.

As a result, we no longer have to distinguish
check and inventory plugins. In particular the
callsite in cmk.base.core_nagios can be simpified
significantly as a result of that.

Change-Id: Ib5a16a926270c9ceefce1070d8c42907c1e5ac11

cmk/base/api/agent_based/register/__init__.py

@@ -11,7 +11,6 @@
     get_discovery_ruleset,
     get_host_label_ruleset,
     get_previously_loaded_plugins,
-    get_relevant_raw_sections,
     get_section_producers,
     is_stored_ruleset,
     iter_all_discovery_rulesets,
@@ -21,6 +20,7 @@
     set_host_label_ruleset,
 )
 from ._discover import load_all_plugins, load_selected_plugins
+from .utils import filter_relevant_raw_sections
 
 __all__ = [
     "add_discovery_ruleset",
@@ -30,7 +30,7 @@
     "get_discovery_ruleset",
     "get_host_label_ruleset",
     "get_previously_loaded_plugins",
-    "get_relevant_raw_sections",
+    "filter_relevant_raw_sections",
     "get_section_producers",
     "is_stored_ruleset",
     "iter_all_discovery_rulesets",

cmk/base/api/agent_based/register/_config.py

@@ -129,29 +129,6 @@ def get_inventory_plugin(plugin_name: InventoryPluginName) -> InventoryPlugin |
     return registered_inventory_plugins.get(plugin_name)
 
 
-def get_relevant_raw_sections(
-    *,
-    check_plugin_names: Iterable[CheckPluginName],
-    inventory_plugin_names: Iterable[InventoryPluginName],
-) -> dict[SectionName, SectionPlugin]:
-    """return the raw sections potentially relevant for the given check or inventory plugins"""
-    parsed_section_names: set[ParsedSectionName] = set()
-
-    for check_plugin_name in check_plugin_names:
-        if check_plugin := get_check_plugin(check_plugin_name):
-            parsed_section_names.update(check_plugin.sections)
-
-    for inventory_plugin_name in inventory_plugin_names:
-        if inventory_plugin := get_inventory_plugin(inventory_plugin_name):
-            parsed_section_names.update(inventory_plugin.sections)
-
-    return {
-        section_name: section
-        for parsed_name in parsed_section_names
-        for section_name, section in _sections_by_parsed_name[parsed_name].items()
-    }
-
-
 def get_section_plugin(section_name: SectionName) -> SectionPlugin | None:
     return registered_agent_sections.get(section_name) or registered_snmp_sections.get(section_name)
 

cmk/base/api/agent_based/register/snmp_plugin_store.py

@@ -3,6 +3,8 @@
 # This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
 # conditions defined in the file COPYING, which is part of this source code package.
 
+import itertools
+
 from cmk.utils.sectionname import SectionName
 
 from cmk.snmplib import (
@@ -17,7 +19,9 @@
 
 from ._config import (
     AgentBasedPlugins,
-    get_relevant_raw_sections,
+)
+from .utils import (
+    filter_relevant_raw_sections,
 )
 
 __all__ = ["make_plugin_store"]
@@ -26,9 +30,11 @@
 def _make_inventory_sections(plugins: AgentBasedPlugins) -> frozenset[SectionName]:
     return frozenset(
         s
-        for s in get_relevant_raw_sections(  # this will need to know all the plugins soon.
-            check_plugin_names=(),
-            inventory_plugin_names=plugins.inventory_plugins,
+        for s in filter_relevant_raw_sections(
+            consumers=plugins.inventory_plugins.values(),
+            sections=itertools.chain(
+                plugins.agent_sections.values(), plugins.snmp_sections.values()
+            ),
         )
         if s in plugins.snmp_sections
     )

cmk/base/api/agent_based/register/utils.py

@@ -6,19 +6,20 @@
 # pylint: disable=protected-access
 import inspect
 import sys
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Callable, Iterable, Mapping, Sequence
 from typing import Final, get_args, Literal, NoReturn, Union
 
 from cmk.ccc.version import Edition
 
 from cmk.utils.check_utils import ParametersTypeAlias
 from cmk.utils.rulesets import RuleSetName
+from cmk.utils.sectionname import SectionName
 
 from cmk.checkengine.checking import CheckPluginName
 from cmk.checkengine.inventory import InventoryPluginName
 from cmk.checkengine.sectionparser import ParsedSectionName
 
-from cmk.base.api.agent_based.plugin_classes import CheckPlugin
+from cmk.base.api.agent_based.plugin_classes import CheckPlugin, InventoryPlugin, SectionPlugin
 
 from cmk.agent_based.v1.register import RuleSetType
 from cmk.discover_plugins import PluginLocation
@@ -237,3 +238,20 @@ def validate_check_ruleset_item_consistency(
             "At least one of the checks in this group needs to be changed "
             f"(offending plug-in: {check_plugin.name}, present plug-ins: {present_plugins})."
         )
+
+
+def filter_relevant_raw_sections(
+    *,
+    consumers: Iterable[CheckPlugin | InventoryPlugin],
+    sections: Iterable[SectionPlugin],
+) -> Mapping[SectionName, SectionPlugin]:
+    """Return the raw sections potentially relevant for the given check or inventory plugins"""
+    parsed_section_names = {
+        section_name for plugin in consumers for section_name in plugin.sections
+    }
+
+    return {
+        section.name: section
+        for section in sections
+        if section.parsed_section_name in parsed_section_names
+    }

cmk/base/config.py

@@ -1942,13 +1942,19 @@ def make_checking_sections(
             checking_sections = selected_sections
         else:
             checking_sections = frozenset(
-                agent_based_register.get_relevant_raw_sections(
-                    check_plugin_names=self.check_table(
-                        hostname,
-                        filter_mode=FilterMode.INCLUDE_CLUSTERED,
-                        skip_ignored=True,
-                    ).needed_check_names(),
-                    inventory_plugin_names=(),
+                agent_based_register.filter_relevant_raw_sections(
+                    consumers=[
+                        plugins.check_plugins[n]
+                        for n in self.check_table(
+                            hostname,
+                            filter_mode=FilterMode.INCLUDE_CLUSTERED,
+                            skip_ignored=True,
+                        ).needed_check_names()
+                        if n in plugins.check_plugins
+                    ],
+                    sections=itertools.chain(
+                        plugins.agent_sections.values(), plugins.snmp_sections.values()
+                    ),
                 )
             )
         return frozenset(s for s in checking_sections if s in plugins.snmp_sections)

cmk/base/core_nagios/_precompile_host_checks.py

@@ -18,7 +18,7 @@
 import re
 import socket
 import sys
-from collections.abc import Collection, Iterable
+from collections.abc import Iterable
 from pathlib import Path
 from typing import assert_never
 
@@ -34,12 +34,14 @@
 from cmk.utils.ip_lookup import IPStackConfig
 from cmk.utils.log import console
 
-from cmk.checkengine.checking import CheckPluginName
-from cmk.checkengine.inventory import InventoryPluginName
-
 import cmk.base.api.agent_based.register as agent_based_register
 import cmk.base.utils
-from cmk.base.api.agent_based.plugin_classes import LegacyPluginLocation, SectionPlugin
+from cmk.base.api.agent_based.plugin_classes import (
+    CheckPlugin,
+    InventoryPlugin,
+    LegacyPluginLocation,
+    SectionPlugin,
+)
 from cmk.base.config import ConfigCache, FilterMode, lookup_ip_address, save_packed_config
 
 from cmk.discover_plugins import PluginLocation
@@ -248,76 +250,60 @@ def _make_needed_plugins_locations(
     list[PluginLocation],
     list[str],  # TODO: change this to `LegacyPluginLocation` once the special agents are migrated
 ]:
-    (
-        needed_agent_based_check_plugin_names,
-        needed_agent_based_inventory_plugin_names,
-    ) = _get_needed_plugin_names(config_cache, hostname, plugins.inventory_plugins)
+    needed_agent_based_plugins = _get_needed_plugins(config_cache, hostname, plugins)
 
     if hostname in config_cache.hosts_config.clusters:
         assert config_cache.nodes(hostname)
         for node in config_cache.nodes(hostname):
-            (
-                node_needed_agent_based_check_plugin_names,
-                node_needed_agent_based_inventory_plugin_names,
-            ) = _get_needed_plugin_names(config_cache, node, plugins.inventory_plugins)
-            needed_agent_based_check_plugin_names.update(node_needed_agent_based_check_plugin_names)
-            needed_agent_based_inventory_plugin_names.update(
-                node_needed_agent_based_inventory_plugin_names
-            )
+            # we're deduplicating later.
+            needed_agent_based_plugins.extend(_get_needed_plugins(config_cache, node, plugins))
 
     needed_legacy_special_agents = _get_needed_legacy_special_agents(config_cache, hostname)
 
-    if not any(
-        (
-            needed_legacy_special_agents,
-            needed_agent_based_check_plugin_names,
-            needed_agent_based_inventory_plugin_names,
-        )
-    ):
-        return [], []
-
-    needed_agent_based_section_plugin_names = agent_based_register.get_relevant_raw_sections(
-        check_plugin_names=needed_agent_based_check_plugin_names,
-        inventory_plugin_names=needed_agent_based_inventory_plugin_names,
-    )
+    needed_agent_based_sections = agent_based_register.filter_relevant_raw_sections(
+        consumers=needed_agent_based_plugins,
+        sections=itertools.chain(plugins.agent_sections.values(), plugins.snmp_sections.values()),
+    ).values()
 
     return (
         _get_needed_agent_based_locations(
-            needed_agent_based_section_plugin_names.values(),
-            needed_agent_based_check_plugin_names,
-            needed_agent_based_inventory_plugin_names,
-            plugins,
+            itertools.chain(needed_agent_based_sections, needed_agent_based_plugins)
         ),
         _get_needed_legacy_check_files(
-            needed_agent_based_section_plugin_names.values(),
-            needed_agent_based_check_plugin_names,
+            itertools.chain(needed_agent_based_sections, needed_agent_based_plugins),
             needed_legacy_special_agents,
         ),
     )
 
 
-def _get_needed_plugin_names(
+def _get_needed_plugins(
     config_cache: ConfigCache,
     host_name: HostName,
-    inventory_plugins: Collection[InventoryPluginName],
-) -> tuple[set[CheckPluginName], set[InventoryPluginName]]:
+    agent_based_plugins: agent_based_register.AgentBasedPlugins,
+) -> list[CheckPlugin | InventoryPlugin]:
     # Collect the needed check plug-in names using the host check table.
     # Even auto-migrated checks must be on the list of needed *agent based* plugins:
     # In those cases, the module attribute will be `None`, so nothing will
     # be imported; BUT: we need it in the list, because it must be considered
     # when determining the needed *section* plugins.
     # This matters in cases where the section is migrated, but the check
     # plugins are not.
-    return (
-        config_cache.check_table(
-            host_name,
-            filter_mode=FilterMode.INCLUDE_CLUSTERED,
-            skip_ignored=False,
-        ).needed_check_names(),
-        set(inventory_plugins)
-        if config_cache.hwsw_inventory_parameters(host_name).status_data_inventory
-        else set(),
-    )
+    return [
+        *(
+            plugin
+            for name in config_cache.check_table(
+                host_name,
+                filter_mode=FilterMode.INCLUDE_CLUSTERED,
+                skip_ignored=False,
+            ).needed_check_names()
+            if (plugin := agent_based_plugins.check_plugins.get(name))
+        ),
+        *(
+            agent_based_plugins.inventory_plugins.values()
+            if config_cache.hwsw_inventory_parameters(host_name).status_data_inventory
+            else ()
+        ),
+    ]
 
 
 def _get_needed_legacy_special_agents(config_cache: ConfigCache, host_name: HostName) -> set[str]:
@@ -332,23 +318,13 @@ def _get_needed_legacy_special_agents(config_cache: ConfigCache, host_name: Host
 
 
 def _get_needed_legacy_check_files(
-    section_plugins: Iterable[SectionPlugin],
-    check_plugin_names: Iterable[CheckPluginName],
+    # Note: we don't *have* any InventoryPlugins that match the condition below, but
+    # it's easier to type it like this.
+    plugins: Iterable[SectionPlugin | CheckPlugin | InventoryPlugin],
     legacy_special_agent_names: set[str],
 ) -> list[str]:
     return sorted(
-        {
-            section.location.file_name
-            for section in section_plugins
-            if isinstance(section.location, LegacyPluginLocation)
-        }
-        | {
-            check_plugin.location.file_name
-            for check_plugin_name in check_plugin_names
-            if (check_plugin := agent_based_register.get_check_plugin(check_plugin_name))
-            is not None
-            and isinstance(check_plugin.location, LegacyPluginLocation)
-        }
+        {p.location.file_name for p in plugins if isinstance(p.location, LegacyPluginLocation)}
         | {
             f"{os.path.join(base, name)}.py"
             for base in (cmk.utils.paths.local_checks_dir, cmk.utils.paths.checks_dir)
@@ -358,26 +334,7 @@ def _get_needed_legacy_check_files(
 
 
 def _get_needed_agent_based_locations(
-    section_plugins: Iterable[SectionPlugin],
-    check_plugin_names: Iterable[CheckPluginName],
-    inventory_plugin_names: Iterable[InventoryPluginName],
-    plugins: agent_based_register.AgentBasedPlugins,
+    plugins: Iterable[SectionPlugin | CheckPlugin | InventoryPlugin],
 ) -> list[PluginLocation]:
-    modules = {
-        plugin.location
-        for plugin in [agent_based_register.get_check_plugin(p) for p in check_plugin_names]
-        if plugin is not None and isinstance(plugin.location, PluginLocation)
-    }
-    modules.update(
-        plugin.location
-        for name in inventory_plugin_names
-        if (plugin := plugins.inventory_plugins.get(name)) is not None
-        and isinstance(plugin.location, PluginLocation)
-    )
-    modules.update(
-        section.location
-        for section in section_plugins
-        if isinstance(section.location, PluginLocation)
-    )
-
+    modules = {plugin.location for plugin in plugins if isinstance(plugin.location, PluginLocation)}
     return sorted(modules, key=lambda l: (l.module, l.name or ""))