Callback plugins: show type, render indexes of callback plugins per type (#90)

* Show callback type, and render callback type indexes.

* Undo changes necessary for #65.

* Update tests.

* Improve callback docs.

* Apply suggestions from code review

Co-authored-by: Sandra McCann <samccann@redhat.com>

Co-authored-by: Sandra McCann <samccann@redhat.com>

Author: felixfontein

changelogs/fragments/90-callbacks.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - "Show callback plugin type on callback plugin pages. Also write callback indexes by callback plugin type (https://github.com/ansible-community/antsibull-docs/issues/89, https://github.com/ansible-community/antsibull-docs/pull/90)."

src/antsibull_docs/cli/doc_commands/stable.py

@@ -41,7 +41,11 @@
 from ...utils.collection_name_transformer import CollectionNameTransformer
 from ...write_docs.collections import output_extra_docs, output_indexes
 from ...write_docs.hierarchy import output_collection_index, output_collection_namespace_indexes
-from ...write_docs.indexes import output_environment_variables, output_plugin_indexes
+from ...write_docs.indexes import (
+    output_callback_indexes,
+    output_environment_variables,
+    output_plugin_indexes,
+)
 from ...write_docs.plugin_stubs import output_all_plugin_stub_rst
 from ...write_docs.plugins import output_all_plugin_rst
 
@@ -269,6 +273,35 @@ def get_plugin_contents(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
     return plugin_contents
 
 
+def get_callback_plugin_contents(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
+                                 ) -> t.DefaultDict[str, t.DefaultDict[str, t.Dict[str, str]]]:
+    """
+    Return the collections with their plugins for every callback plugin type.
+
+    :arg plugin_info: Mapping of plugin type to a mapping of plugin name to plugin record.
+        The plugin_type, plugin_name, and short_description from plugin_records are used.
+    :returns: A Mapping of callback plugin type to a mapping of collection name to a mapping of
+        plugin names to short_descriptions.
+    callback_type:
+        collection:
+            - plugin_short_name: short_description
+    """
+    callback_plugin_contents: t.DefaultDict[str, t.DefaultDict[str, t.Dict[str, str]]]
+    callback_plugin_contents = defaultdict(lambda: defaultdict(dict))
+
+    if plugin_info.get('callback'):
+        for plugin_name, plugin_desc in plugin_info['callback'].items():
+            if 'doc' in plugin_desc:
+                desc = plugin_desc['doc'].get('short_description') or ''
+                callback_type = plugin_desc['doc'].get('type') or ''
+                if callback_type:
+                    namespace, collection, short_name = get_fqcn_parts(plugin_name)
+                    collection_name = '.'.join((namespace, collection))
+                    callback_plugin_contents[callback_type][collection_name][short_name] = desc
+
+    return callback_plugin_contents
+
+
 def get_collection_contents(plugin_content: t.Mapping[str, t.Mapping[str, t.Mapping[str, str]]],
                             ) -> t.DefaultDict[str, t.Dict[str, t.Mapping[str, str]]]:
     """
@@ -369,6 +402,7 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
     flog.debug('Finished getting collection link data')
 
     plugin_contents = get_plugin_contents(new_plugin_info, nonfatal_errors)
+    callback_plugin_contents = get_callback_plugin_contents(new_plugin_info)
     collection_to_plugin_info = get_collection_contents(plugin_contents)
     # Make sure collections without documentable plugins are mentioned
     for collection in collection_metadata:
@@ -414,6 +448,11 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
                                           collection_install=collection_install,
                                           for_official_docsite=for_official_docsite))
         flog.notice('Finished writing plugin indexes')
+        asyncio_run(output_callback_indexes(callback_plugin_contents,
+                                            dest_dir, collection_url=collection_url,
+                                            collection_install=collection_install,
+                                            for_official_docsite=for_official_docsite))
+        flog.notice('Finished writing callback plugin indexes')
 
     asyncio_run(output_indexes(collection_to_plugin_info, dest_dir,
                                collection_url=collection_url,

src/antsibull_docs/data/docsite/list_of_callback_plugins.rst.j2

@@ -0,0 +1,26 @@
+{#
+  Copyright (c) Ansible Project
+  GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+  SPDX-License-Identifier: GPL-3.0-or-later
+#}
+
+:orphan:
+
+.. _list_of_@{ callback_type }@_callback_plugins:
+
+Index of all @{ callback_type | capitalize }@ Callback Plugins
+=============@{ '=' * (callback_type | length) }@=================
+
+See :ref:`list_of_callback_plugins` for the list of *all* callback plugins.
+
+{% for collection_name, plugins in per_collection_plugins.items() | sort %}
+@{ collection_name }@
+@{ '-' * (collection_name | length) }@
+
+{%   for plugin_name, plugin_desc in plugins.items() | sort %}
+* :ref:`@{ collection_name }@.@{ plugin_name }@ <ansible_collections.@{ collection_name }@.@{ plugin_name }@_callback>` -- @{ plugin_desc | rst_ify }@
+{%   endfor %}
+
+{% else %}
+No public @{ callback_type }@ callback plugin found.
+{% endfor %}

src/antsibull_docs/data/docsite/list_of_plugins.rst.j2

@@ -18,6 +18,16 @@ Index of all Roles
 Index of all @{ plugin_type | capitalize }@ Plugins
 =============@{ '=' * (plugin_type | length) }@========
 {% endif %}
+{% if plugin_type == 'callback' %}
+
+.. toctree::
+   :maxdepth: 1
+   :caption: List of callback plugins by callback type
+   :glob:
+
+   callback_index_*
+
+{% endif %}
 
 {% for collection_name, plugins in per_collection_plugins.items() | sort %}
 @{ collection_name }@

src/antsibull_docs/data/docsite/plugin.rst.j2

@@ -139,6 +139,20 @@ DEPRECATED
 :Alternative: @{ doc['deprecated']['alternative'] | rst_ify }@
 {% endif %}
 
+{% if plugin_type == 'callback' %}
+Callback plugin
+---------------
+
+{%   if doc['type'] == 'stdout' %}
+This plugin is a **stdout callback**. You can use only use one stdout callback at a time. Additional aggregate or notification callbacks can be enabled though.
+{%   elif doc['type'] == 'aggregate' %}
+This plugin is an **aggregate callback**. It adds additional console output next to the configured stdout callback.
+{%   elif doc['type'] == 'notification' %}
+This plugin is a **notification callback**. It sends information for a playbook run to other applications, services, or systems.
+{%   endif %}
+See :ref:`callback_plugins` for more information on callback plugins.
+
+{% endif %}
 Synopsis
 --------
 

src/antsibull_docs/write_docs/indexes.py

@@ -24,6 +24,33 @@
 mlog = log.fields(mod=__name__)
 
 
+async def write_callback_type_index(callback_type: str,
+                                    per_collection_plugins: t.Mapping[str, t.Mapping[str, str]],
+                                    template: Template,
+                                    dest_filename: str,
+                                    for_official_docsite: bool = False) -> None:
+    """
+    Write an index page for each plugin type.
+
+    :arg callback_type: The callback plugin type to write the index for.
+    :arg per_collection_plugins: Mapping of collection_name to Mapping of plugin_name to
+        short_description.
+    :arg template: A template to render the plugin index.
+    :arg dest_filename: The destination filename.
+    :kwarg for_official_docsite: Default False.  Set to True to use wording specific for the
+        official docsite on docs.ansible.com.
+    """
+    index_contents = _render_template(
+        template,
+        dest_filename,
+        callback_type=callback_type,
+        per_collection_plugins=per_collection_plugins,
+        for_official_docsite=for_official_docsite,
+    )
+
+    await write_file(dest_filename, index_contents)
+
+
 async def write_plugin_type_index(plugin_type: str,
                                   per_collection_plugins: t.Mapping[str, t.Mapping[str, str]],
                                   template: Template,
@@ -51,6 +78,53 @@ async def write_plugin_type_index(plugin_type: str,
     await write_file(dest_filename, index_contents)
 
 
+async def output_callback_indexes(plugin_info: PluginCollectionInfoT,
+                                  dest_dir: str,
+                                  collection_url: CollectionNameTransformer,
+                                  collection_install: CollectionNameTransformer,
+                                  for_official_docsite: bool = False) -> None:
+    """
+    Generate top-level callback plugin index pages for all callback plugins of a type in all
+    collections.
+
+    :arg plugin_info: Mapping of callback_type to Mapping of collection_name to Mapping of
+        plugin_name to short_description.
+    :arg dest_dir: The directory to place the documentation in.
+    :kwarg for_official_docsite: Default False.  Set to True to use wording specific for the
+        official docsite on docs.ansible.com.
+    """
+    flog = mlog.fields(func='output_callback_indexes')
+    flog.debug('Enter')
+
+    env = doc_environment(
+        ('antsibull_docs.data', 'docsite'),
+        collection_url=collection_url,
+        collection_install=collection_install)
+    # Get the templates
+    plugin_list_tmpl = env.get_template('list_of_callback_plugins.rst.j2')
+
+    collection_toplevel = os.path.join(dest_dir, 'collections')
+    flog.fields(toplevel=collection_toplevel, exists=os.path.isdir(collection_toplevel)).debug(
+        'collection_toplevel exists?')
+    # This is only safe because we made sure that the top of the directory tree we're writing to
+    # (docs/docsite/rst) is only writable by us.
+    os.makedirs(collection_toplevel, mode=0o755, exist_ok=True)
+
+    writers = []
+    lib_ctx = app_context.lib_ctx.get()
+    async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool:
+        for callback_type, per_collection_data in plugin_info.items():
+            filename = os.path.join(collection_toplevel, f'callback_index_{callback_type}.rst')
+            writers.append(await pool.spawn(
+                write_callback_type_index(
+                    callback_type, per_collection_data, plugin_list_tmpl,
+                    filename, for_official_docsite=for_official_docsite)))
+
+        await asyncio.gather(*writers)
+
+    flog.debug('Leave')
+
+
 async def output_plugin_indexes(plugin_info: PluginCollectionInfoT,
                                 dest_dir: str,
                                 collection_url: CollectionNameTransformer,

tests/functional/baseline-default/collections/callback_index_stdout.rst

@@ -0,0 +1,15 @@
+
+:orphan:
+
+.. _list_of_stdout_callback_plugins:
+
+Index of all Stdout Callback Plugins
+====================================
+
+See :ref:`list_of_callback_plugins` for the list of *all* callback plugins.
+
+ns2.col
+-------
+
+* :ref:`ns2.col.foo <ansible_collections.ns2.col.foo_callback>` -- Foo output
+

tests/functional/baseline-default/collections/index_callback.rst

@@ -6,6 +6,14 @@
 Index of all Callback Plugins
 =============================
 
+.. toctree::
+   :maxdepth: 1
+   :caption: List of callback plugins by callback type
+   :glob:
+
+   callback_index_*
+
+
 ns2.col
 -------
 

tests/functional/baseline-default/collections/ns2/col/foo_callback.rst

@@ -62,6 +62,12 @@ New in ns2.col 0.0.1
 .. Deprecated
 
 
+Callback plugin
+---------------
+
+This plugin is a **stdout callback**. You can use only use one stdout callback at a time. Additional aggregate or notification callbacks can be enabled though.
+See :ref:`callback_plugins` for more information on callback plugins.
+
 Synopsis
 --------
 

tests/functional/baseline-no-breadcrumbs/collections/callback_index_stdout.rst

@@ -0,0 +1,15 @@
+
+:orphan:
+
+.. _list_of_stdout_callback_plugins:
+
+Index of all Stdout Callback Plugins
+====================================
+
+See :ref:`list_of_callback_plugins` for the list of *all* callback plugins.
+
+ns2.col
+-------
+
+* :ref:`ns2.col.foo <ansible_collections.ns2.col.foo_callback>` -- Foo output
+