Extended plugin docs lint (#12)

* Move load_collection_name to new module.

* Call plugin docs linter.

* Allow to disable plugin docs linting.

* Add --plugin-docs flag for lint-collection-docs.

* Make new code work with rstcheck 6.

* Linting.

Author: felixfontein

changelogs/fragments/12-lint-collection-docs-plugin-rst.yml

@@ -0,0 +1,4 @@
+minor_changes:
+  - "The ``lint-collection-docs`` subcommand has a new boolean flag ``--plugin-docs`` which renders the plugin docs
+     to RST and validates them with rstcheck. This can be used as a lighter version of rendering the docsite in CI
+     (https://github.com/ansible-community/antsibull-docs/pull/12)."

src/antsibull_docs/cli/antsibull_docs.py

@@ -360,6 +360,12 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace:
                                              metavar='/path/to/collection',
                                              help='path to collection (directory that includes'
                                              ' galaxy.yml)')
+    lint_collection_docs_parser.add_argument('--plugin-docs',
+                                             dest='plugin_docs', action=BooleanOptionalAction,
+                                             default=False,
+                                             help='Determine whether to also check RST file'
+                                             ' generation and validation for plugins and roles'
+                                             ' in this collection. (default: True)')
 
     flog.debug('Argument parser setup')
 

src/antsibull_docs/cli/doc_commands/lint_collection_docs.py

@@ -8,6 +8,7 @@
 
 from ...collection_links import lint_collection_links
 from ...lint_extra_docs import lint_collection_extra_docs_files
+from ...lint_plugin_docs import lint_collection_plugin_docs
 
 
 mlog = log.fields(mod=__name__)
@@ -26,13 +27,18 @@ def lint_collection_docs() -> int:
     app_ctx = app_context.app_ctx.get()
 
     collection_root = app_ctx.extra['collection_root_path']
+    plugin_docs = app_ctx.extra['plugin_docs']
 
     flog.notice('Linting extra docs files')
     errors = lint_collection_extra_docs_files(collection_root)
 
     flog.notice('Linting collection links')
     errors.extend(lint_collection_links(collection_root))
 
+    if plugin_docs:
+        flog.notice('Linting plugin docs')
+        errors.extend(lint_collection_plugin_docs(collection_root))
+
     messages = sorted(set(f'{error[0]}:{error[1]}:{error[2]}: {error[3]}' for error in errors))
 
     for message in messages:

src/antsibull_docs/docs_parsing/ansible_doc.py

@@ -184,6 +184,31 @@ def _extract_ansible_builtin_metadata(stdout: str) -> AnsibleCollectionMetadata:
     return AnsibleCollectionMetadata(path=path, version=version)
 
 
+def parse_ansible_galaxy_collection_list(raw_output: str,
+                                         collection_names: t.Optional[t.List[str]] = None,
+                                         ) -> t.List[t.Tuple[str, str, str, t.Optional[str]]]:
+    result = []
+    current_base_path = None
+    for line in raw_output.splitlines():
+        parts = line.split()
+        if len(parts) >= 2:
+            if parts[0] == '#':
+                current_base_path = parts[1]
+            elif current_base_path is not None:
+                collection_name = parts[0]
+                version = parts[1]
+                if '.' in collection_name:
+                    if collection_names is None or collection_name in collection_names:
+                        namespace, name = collection_name.split('.', 2)
+                        result.append((
+                            namespace,
+                            name,
+                            os.path.join(current_base_path, namespace, name),
+                            None if version == '*' else version
+                        ))
+    return result
+
+
 def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
                             env: t.Dict[str, str],
                             collection_names: t.Optional[t.List[str]] = None,
@@ -201,21 +226,9 @@ def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
     venv_ansible_galaxy = venv.get_command('ansible-galaxy')
     ansible_collection_list_cmd = venv_ansible_galaxy('collection', 'list', _env=env)
     raw_result = ansible_collection_list_cmd.stdout.decode('utf-8', errors='surrogateescape')
-    current_base_path = None
-    for line in raw_result.splitlines():
-        parts = line.split()
-        if len(parts) >= 2:
-            if parts[0] == '#':
-                current_base_path = parts[1]
-            else:
-                collection_name = parts[0]
-                version = parts[1]
-                if '.' in collection_name:
-                    if collection_names is None or collection_name in collection_names:
-                        namespace, name = collection_name.split('.', 2)
-                        collection_metadata[collection_name] = AnsibleCollectionMetadata(
-                            path=os.path.join(current_base_path, namespace, name),
-                            version=None if version == '*' else version)
+    for namespace, name, path, version in parse_ansible_galaxy_collection_list(raw_result):
+        collection_metadata[f'{namespace}.{name}'] = AnsibleCollectionMetadata(
+            path=path, version=version)
 
     return collection_metadata
 

src/antsibull_docs/lint_extra_docs.py

@@ -1,17 +1,14 @@
 # coding: utf-8
-# Author: Felix Fontein <[email protected]>
+# Author: Felix Fontein <[email protected]>
 # License: GPLv3+
 # Copyright: Ansible Project, 2021
 """Lint extra collection documentation in docs/docsite/."""
 
-import json
 import os
 import os.path
 import re
 import typing as t
 
-from antsibull_core.yaml import load_yaml_file
-
 from .extra_docs import (
     find_extra_docs,
     lint_required_conditions,
@@ -20,28 +17,12 @@
 )
 from .rstcheck import check_rst_content
 
+from .lint_helpers import (
+    load_collection_name,
+)
 
-_RST_LABEL_DEFINITION = re.compile(r'''^\.\. _([^:]+):''')
-
-
-def load_collection_name(path_to_collection: str) -> str:
-    '''Load collection name (namespace.name) from collection's galaxy.yml.'''
-    manifest_json_path = os.path.join(path_to_collection, 'MANIFEST.json')
-    if os.path.isfile(manifest_json_path):
-        with open(manifest_json_path, 'rb') as f:
-            manifest_json = json.load(f)
-        # pylint:disable-next=consider-using-f-string
-        collection_name = '{namespace}.{name}'.format(**manifest_json['collection_info'])
-        return collection_name
-
-    galaxy_yml_path = os.path.join(path_to_collection, 'galaxy.yml')
-    if os.path.isfile(galaxy_yml_path):
-        galaxy_yml = load_yaml_file(galaxy_yml_path)
-        # pylint:disable-next=consider-using-f-string
-        collection_name = '{namespace}.{name}'.format(**galaxy_yml)
-        return collection_name
 
-    raise Exception(f'Cannot find files {manifest_json_path} and {galaxy_yml_path}')
+_RST_LABEL_DEFINITION = re.compile(r'''^\.\. _([^:]+):''')
 
 
 # pylint:disable-next=unused-argument

src/antsibull_docs/lint_helpers.py

@@ -0,0 +1,36 @@
+# coding: utf-8
+# Author: Felix Fontein <[email protected]>
+# License: GPLv3+
+# Copyright: Ansible Project, 2022
+"""Lint plugin docs."""
+
+import json
+import os
+import os.path
+import typing as t
+
+from antsibull_core.yaml import load_yaml_file
+
+
+def load_collection_info(path_to_collection: str) -> t.Dict[str, t.Any]:
+    '''Load collection name (namespace.name) from collection's galaxy.yml.'''
+    manifest_json_path = os.path.join(path_to_collection, 'MANIFEST.json')
+    if os.path.isfile(manifest_json_path):
+        with open(manifest_json_path, 'rb') as f:
+            manifest_json = json.load(f)
+        return manifest_json['collection_info']
+
+    galaxy_yml_path = os.path.join(path_to_collection, 'galaxy.yml')
+    if os.path.isfile(galaxy_yml_path):
+        galaxy_yml = load_yaml_file(galaxy_yml_path)
+        return galaxy_yml
+
+    raise Exception(f'Cannot find files {manifest_json_path} and {galaxy_yml_path}')
+
+
+def load_collection_name(path_to_collection: str) -> str:
+    '''Load collection name (namespace.name) from collection's galaxy.yml.'''
+    info = load_collection_info(path_to_collection)
+    # pylint:disable-next=consider-using-f-string
+    collection_name = '{namespace}.{name}'.format(**info)
+    return collection_name

src/antsibull_docs/lint_plugin_docs.py

@@ -0,0 +1,191 @@
+# coding: utf-8
+# Author: Felix Fontein <[email protected]>
+# License: GPLv3+
+# Copyright: Ansible Project, 2022
+"""Lint plugin docs."""
+
+import os
+import shutil
+import tempfile
+import typing as t
+
+import sh
+
+from antsibull_core.compat import asyncio_run
+from antsibull_core.venv import FakeVenvRunner
+
+from .lint_helpers import (
+    load_collection_info,
+)
+
+from .docs_parsing.ansible_doc import (
+    parse_ansible_galaxy_collection_list,
+)
+
+from .augment_docs import augment_docs
+from .cli.doc_commands.stable import (
+    normalize_all_plugin_info,
+    get_plugin_contents,
+    get_collection_contents,
+)
+from .collection_links import load_collections_links
+from .docs_parsing.parsing import get_ansible_plugin_info
+from .docs_parsing.routing import (
+    load_all_collection_routing,
+    remove_redirect_duplicates,
+)
+from .jinja2.environment import doc_environment
+from .write_docs import create_plugin_rst
+from .rstcheck import check_rst_content
+
+
+class CollectionCopier:
+    dir: t.Optional[str]
+
+    def __init__(self):
+        self.dir = None
+
+    def __enter__(self):
+        if self.dir is None:
+            raise AssertionError('Collection copier already initialized')
+        self.dir = os.path.realpath(tempfile.mkdtemp(prefix='antsibull-docs-'))
+        return self
+
+    def add_collection(self, collecion_source_path: str, namespace: str, name: str) -> None:
+        self_dir = self.dir
+        if self_dir is None:
+            raise AssertionError('Collection copier not initialized')
+        collection_container_dir = os.path.join(
+            self_dir, 'ansible_collections', namespace)
+        os.makedirs(collection_container_dir, exist_ok=True)
+
+        collection_dir = os.path.join(collection_container_dir, name)
+        shutil.copytree(collecion_source_path, collection_dir, symlinks=True)
+
+    def __exit__(self, type_, value, traceback_):
+        self_dir = self.dir
+        if self_dir is None:
+            raise AssertionError('Collection copier not initialized')
+        shutil.rmtree(self_dir, ignore_errors=True)
+        self.dir = None
+
+
+class CollectionFinder:
+    def __init__(self):
+        self.collections = {}
+        stdout = sh.Command('ansible-galaxy')('collection', 'list').stdout
+        raw_output = stdout.decode('utf-8', errors='surrogateescape')
+        for namespace, name, path, _ in reversed(parse_ansible_galaxy_collection_list(raw_output)):
+            self.collections[f'{namespace}.{name}'] = path
+
+    def find(self, namespace, name):
+        return self.collections.get(f'{namespace}.{name}')
+
+
+def _lint_collection_plugin_docs(collections_dir: str, collection_name: str,
+                                 original_path_to_collection: str,
+                                 ) -> t.List[t.Tuple[str, int, int, str]]:
+    # Load collection docs
+    venv = FakeVenvRunner()
+    plugin_info, collection_metadata = asyncio_run(get_ansible_plugin_info(
+        venv, collections_dir, collection_names=[collection_name]))
+    # Load routing information
+    collection_routing = asyncio_run(load_all_collection_routing(collection_metadata))
+    # Process data
+    remove_redirect_duplicates(plugin_info, collection_routing)
+    plugin_info, nonfatal_errors = asyncio_run(normalize_all_plugin_info(plugin_info))
+    augment_docs(plugin_info)
+    # Load link data
+    link_data = asyncio_run(load_collections_links(
+        {name: data.path for name, data in collection_metadata.items()}))
+    # More processing
+    plugin_contents = get_plugin_contents(plugin_info, nonfatal_errors)
+    collection_to_plugin_info = get_collection_contents(plugin_contents)
+    for collection in collection_metadata:
+        collection_to_plugin_info[collection]  # pylint:disable=pointless-statement
+    # Collect non-fatal errors
+    result = []
+    for plugin_type, plugins in sorted(nonfatal_errors.items()):
+        for plugin_name, errors in sorted(plugins.items()):
+            for error in errors:
+                result.append((
+                    os.path.join(original_path_to_collection, 'plugins', plugin_type, plugin_name),
+                    0,
+                    0,
+                    error,
+                ))
+    # Compose RST files and check for errors
+    # Setup the jinja environment
+    env = doc_environment(('antsibull_docs.data', 'docsite'))
+    # Get the templates
+    plugin_tmpl = env.get_template('plugin.rst.j2')
+    role_tmpl = env.get_template('role.rst.j2')
+    error_tmpl = env.get_template('plugin-error.rst.j2')
+
+    for collection_name_, plugins_by_type in collection_to_plugin_info.items():
+        for plugin_type, plugins in plugins_by_type.items():
+            plugin_type_tmpl = plugin_tmpl
+            if plugin_type == 'role':
+                plugin_type_tmpl = role_tmpl
+            for plugin_short_name, dummy_ in plugins.items():
+                plugin_name = '.'.join((collection_name_, plugin_short_name))
+                rst_content = create_plugin_rst(
+                    collection_name_, collection_metadata[collection_name_],
+                    link_data[collection_name_],
+                    plugin_short_name, plugin_type,
+                    plugin_info[plugin_type].get(plugin_name),
+                    nonfatal_errors[plugin_type][plugin_name],
+                    plugin_type_tmpl, error_tmpl,
+                    use_html_blobs=False,
+                )
+                path = os.path.join(
+                    original_path_to_collection, 'plugins', plugin_type,
+                    f'{plugin_short_name}.rst')
+                rst_results = check_rst_content(
+                    rst_content, filename=path,
+                    ignore_directives=['rst-class'],
+                )
+                result.extend([(path, result[0], result[1], result[2]) for result in rst_results])
+    return result
+
+
+def lint_collection_plugin_docs(path_to_collection: str) -> t.List[t.Tuple[str, int, int, str]]:
+    try:
+        info = load_collection_info(path_to_collection)
+        namespace = info['namespace']
+        name = info['name']
+        dependencies = info.get('dependencies') or {}
+    except Exception:  # pylint:disable=broad-except
+        return [(
+            path_to_collection, 0, 0,
+            'Cannot identify collection with galaxy.yml or MANIFEST.json at this path')]
+    result = []
+    collection_name = f'{namespace}.{name}'
+    done_dependencies = {collection_name}
+    dependencies = sorted(dependencies)
+    with CollectionCopier() as copier:
+        # Copy collection
+        copier.add_collection(path_to_collection, namespace, name)
+        # Copy all dependencies
+        if dependencies:
+            collection_finder = CollectionFinder()
+            while dependencies:
+                dependency = dependencies.pop(0)
+                if dependency in done_dependencies:
+                    continue
+                dep_namespace, dep_name = dependency.split('.', 2)
+                dep_collection_path = collection_finder.find(dep_namespace, dep_name)
+                if dep_collection_path:
+                    copier.add_collection(dep_collection_path, dep_namespace, dep_name)
+                    try:
+                        info = load_collection_info(dep_collection_path)
+                        dependencies.extend(sorted(info.get('dependencies') or {}))
+                    except Exception:  # pylint:disable=broad-except
+                        result.append((
+                            dep_collection_path, 0, 0,
+                            'Cannot identify collection with galaxy.yml or MANIFEST.json'
+                            ' at this path'))
+        # Load docs
+        result.extend(_lint_collection_plugin_docs(
+            copier.dir, collection_name, path_to_collection))
+    return result