Skip to content

Commit 54be8e1

Browse files
authored
Improve environment variable handling (#73)
* Always provide metadata of ansible.builtin. * Improve environment variable handling. Create an environment variable index.
1 parent 55a97a0 commit 54be8e1

34 files changed

+491
-61
lines changed

changelogs/fragments/73-env-vars.yml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
minor_changes:
2+
- "Use correct markup (``envvar`` role) for environment variables. Compile an index of all environment variables used by plugins (https://github.com/ansible-community/antsibull-docs/pull/73)."

src/antsibull_docs/cli/doc_commands/stable.py

+21-4
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
load_all_collection_routing,
3838
remove_redirect_duplicates,
3939
)
40+
from ...env_variables import load_ansible_config, collect_referenced_environment_variables
4041
from ...schemas.docs import DOCS_SCHEMAS
4142
from ...utils.collection_name_transformer import CollectionNameTransformer
4243
from ...write_docs import (
@@ -47,6 +48,7 @@
4748
output_indexes,
4849
output_plugin_indexes,
4950
output_extra_docs,
51+
output_environment_variables,
5052
)
5153

5254
if t.TYPE_CHECKING:
@@ -337,12 +339,16 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
337339
app_ctx = app_context.app_ctx.get()
338340

339341
# Get the info from the plugins
340-
plugin_info, collection_metadata = asyncio_run(get_ansible_plugin_info(
342+
plugin_info, full_collection_metadata = asyncio_run(get_ansible_plugin_info(
341343
venv, collection_dir, collection_names=collection_names))
342344
flog.notice('Finished parsing info from plugins and collections')
343345
# flog.fields(plugin_info=plugin_info).debug('Plugin data')
344346
# flog.fields(
345-
# collection_metadata=collection_metadata).debug('Collection metadata')
347+
# collection_metadata=full_collection_metadata).debug('Collection metadata')
348+
349+
collection_metadata = dict(full_collection_metadata)
350+
if collection_names is not None and 'ansible.builtin' not in collection_names:
351+
del collection_metadata['ansible.builtin']
346352

347353
# Load collection routing information
348354
collection_routing = asyncio_run(load_all_collection_routing(collection_metadata))
@@ -363,7 +369,7 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
363369
{name: data.path for name, data in collection_metadata.items()}))
364370
flog.debug('Finished getting collection extra docs data')
365371

366-
# Load collection extra docs data
372+
# Load collection links data
367373
link_data = asyncio_run(load_collections_links(
368374
{name: data.path for name, data in collection_metadata.items()}))
369375
flog.debug('Finished getting collection link data')
@@ -384,6 +390,10 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
384390
print(f"{plugin_name} {plugin_type}: {textwrap.indent(error, ' ').lstrip()}")
385391
return 1
386392

393+
# Handle environment variables
394+
ansible_config = load_ansible_config(full_collection_metadata['ansible.builtin'])
395+
referenced_env_vars = collect_referenced_environment_variables(new_plugin_info, ansible_config)
396+
387397
collection_namespaces = get_collection_namespaces(collection_to_plugin_info.keys())
388398

389399
collection_url = CollectionNameTransformer(
@@ -444,7 +454,14 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
444454

445455
asyncio_run(output_extra_docs(dest_dir, extra_docs_data,
446456
squash_hierarchy=squash_hierarchy))
447-
flog.debug('Finished writing extra extra docs docs')
457+
flog.debug('Finished writing extra docs')
458+
459+
if referenced_env_vars:
460+
asyncio_run(output_environment_variables(dest_dir, referenced_env_vars,
461+
squash_hierarchy=squash_hierarchy))
462+
flog.debug('Finished writing environment variables')
463+
else:
464+
flog.debug('Skipping environment variables (as there are none)')
448465
return 0
449466

450467

src/antsibull_docs/data/collection-enum.py

+4-5
Original file line numberDiff line numberDiff line change
@@ -249,11 +249,10 @@ def main(args):
249249
collection_name = f'{meta["namespace"]}.{meta["name"]}'
250250
if match_filter(collection_name, coll_filter):
251251
result['collections'][collection_name] = meta
252-
if match_filter('ansible.builtin', coll_filter):
253-
result['collections']['ansible.builtin'] = {
254-
'path': os.path.dirname(ansible_release.__file__),
255-
'version': ansible_release.__version__,
256-
}
252+
result['collections']['ansible.builtin'] = {
253+
'path': os.path.dirname(ansible_release.__file__),
254+
'version': ansible_release.__version__,
255+
}
257256

258257
print(json.dumps(
259258
result, cls=AnsibleJSONEncoder, sort_keys=True, indent=4 if arguments.pretty else None))
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{#
2+
Copyright (c) Ansible Project
3+
GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
4+
SPDX-License-Identifier: GPL-3.0-or-later
5+
#}
6+
7+
:orphan:
8+
9+
.. _list_of_collection_env_vars:
10+
11+
Index of all Collection Environment Variables
12+
=============================================
13+
14+
The following index documents all environment variables declared by plugins in collections.
15+
Environment variables used by the ansible-core configuation are documented in :ref:`ansible_configuration_settings`.
16+
{# TODO: use label `ansible_configuration_env_vars` once the ansible-core PR is merged #}
17+
18+
{% for _, env_var in env_variables | dictsort %}
19+
.. envvar:: @{ env_var.name }@
20+
21+
{% for paragraph in env_var.description or [] %}
22+
@{ paragraph | replace('\n', '\n ') | rst_ify | indent(4) }@
23+
24+
{% endfor %}
25+
*Used by:*
26+
{% set plugins_ = [] %}
27+
{% for plugin_type, plugins in env_var.plugins.items() %}
28+
{% for plugin_name in plugins %}
29+
{% set _ = plugins_.append((plugin_name, plugin_type)) %}
30+
{% endfor %}
31+
{% endfor %}
32+
{% for plugin_name, plugin_type in plugins_ | unique | sort %}
33+
:ref:`@{ plugin_name | rst_escape }@ {% if plugin_type == 'module' %}module{% else %}@{ plugin_type }@ plugin{% endif %} <ansible_collections.@{ plugin_name }@_@{ plugin_type }@>`
34+
{%- if not loop.last -%}
35+
,
36+
{% endif -%}
37+
{%- endfor %}
38+
39+
{% endfor %}

src/antsibull_docs/data/docsite/macros/parameters.rst.j2

+2-2
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@
125125
{% endfor %}
126126
{% endif %}
127127
{% for env in value['env'] %}
128-
- Environment variable: @{ env['name'] | rst_escape }@
128+
- Environment variable: :envvar:`@{ env['name'] | rst_escape(escape_ending_whitespace=true) }@`
129129
{% if env['version_added'] is still_relevant(collection=env['version_added_collection'] or collection) %}
130130

131131
:ansible-option-versionadded:`added in @{ version_added_rst(env['version_added'], env['version_added_collection'] or collection) }@`
@@ -250,7 +250,7 @@
250250
{% endif %}
251251
{% for env in value['env'] %}
252252
<li>
253-
<p>Environment variable: @{ env['name'] | escape }@</p>
253+
<p>Environment variable: <code class="xref std std-envvar literal notranslate">@{ env['name'] | escape }@</code></p>
254254
{% if env['version_added'] is still_relevant(collection=env['version_added_collection'] or collection) %}
255255
<p><span class="ansible-option-versionadded">added in @{ version_added_html(env['version_added'], env['version_added_collection'] or collection) }@</span></p>
256256
{% endif %}

src/antsibull_docs/docs_parsing/ansible_doc.py

+5-6
Original file line numberDiff line numberDiff line change
@@ -217,12 +217,11 @@ def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
217217
) -> t.Dict[str, AnsibleCollectionMetadata]:
218218
collection_metadata = {}
219219

220-
# Obtain ansible.builtin version
221-
if collection_names is None or 'ansible.builtin' in collection_names:
222-
venv_ansible = venv.get_command('ansible')
223-
ansible_version_cmd = venv_ansible('--version', _env=env)
224-
raw_result = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape')
225-
collection_metadata['ansible.builtin'] = _extract_ansible_builtin_metadata(raw_result)
220+
# Obtain ansible.builtin version and path
221+
venv_ansible = venv.get_command('ansible')
222+
ansible_version_cmd = venv_ansible('--version', _env=env)
223+
raw_result = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape')
224+
collection_metadata['ansible.builtin'] = _extract_ansible_builtin_metadata(raw_result)
226225

227226
# Obtain collection versions
228227
venv_ansible_galaxy = venv.get_command('ansible-galaxy')

src/antsibull_docs/docs_parsing/parsing.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,9 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
5353
{information from ansible-doc --json. See the ansible-doc documentation
5454
for more info.}
5555
56-
The second component is a Mapping of collection names to metadata.
57-
56+
The second component is a Mapping of collection names to metadata. The second mapping
57+
always includes the metadata for ansible.builtin, even if it was not explicitly
58+
mentioned in ``collection_names``.
5859
"""
5960
flog = mlog.fields(func='get_ansible_plugin_info')
6061

src/antsibull_docs/env_variables.py

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# Author: Felix Fontein <[email protected]>
2+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
3+
# https://www.gnu.org/licenses/gpl-3.0.txt)
4+
# SPDX-License-Identifier: GPL-3.0-or-later
5+
# SPDX-FileCopyrightText: 2022, Ansible Project
6+
"""Environment variable handling."""
7+
8+
import os
9+
import os.path
10+
import typing as t
11+
12+
from antsibull_core import yaml
13+
14+
from .docs_parsing import AnsibleCollectionMetadata
15+
16+
17+
class EnvironmentVariableInfo:
18+
name: str
19+
description: t.Optional[t.List[str]]
20+
plugins: t.Dict[str, t.List[str]] # maps plugin_type to lists of plugin FQCNs
21+
22+
def __init__(self,
23+
name: str,
24+
description: t.Optional[t.List[str]] = None,
25+
plugins: t.Optional[t.Dict[str, t.List[str]]] = None):
26+
self.name = name
27+
self.description = description
28+
self.plugins = plugins or {}
29+
30+
def __repr__(self):
31+
return f'E({self.name}, description={repr(self.description)}, plugins={self.plugins})'
32+
33+
34+
def load_ansible_config(ansible_builtin_metadata: AnsibleCollectionMetadata
35+
) -> t.Mapping[str, t.Mapping[str, t.Any]]:
36+
"""
37+
Load Ansible base configuration (``lib/ansible/config/base.yml``).
38+
39+
:arg ansible_builtin_metadata: Metadata for the ansible.builtin collection.
40+
:returns: A Mapping of configuration options to information on these options.
41+
"""
42+
return yaml.load_yaml_file(os.path.join(ansible_builtin_metadata.path, 'config', 'base.yml'))
43+
44+
45+
def _find_env_vars(options: t.Mapping[str, t.Mapping[str, t.Any]]
46+
) -> t.Generator[t.Tuple[str, t.Optional[t.List[str]]], None, None]:
47+
for _, option_data in options.items():
48+
if isinstance(option_data.get('env'), list):
49+
description = option_data.get('description')
50+
if isinstance(description, str):
51+
description = [description]
52+
if isinstance(description, list):
53+
description = [str(desc) for desc in description]
54+
else:
55+
description = None
56+
for env_var in option_data['env']:
57+
if isinstance(env_var.get('name'), str):
58+
yield (env_var['name'], description)
59+
if isinstance(option_data.get('suboptions'), dict):
60+
yield from _find_env_vars(option_data['suboptions'])
61+
62+
63+
def _collect_env_vars_and_descriptions(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
64+
core_envs: t.Set[str],
65+
) -> t.Tuple[t.Mapping[str, EnvironmentVariableInfo],
66+
t.Mapping[str, t.List[t.List[str]]]]:
67+
other_variables: t.Dict[str, EnvironmentVariableInfo] = {}
68+
other_variable_description: t.Dict[str, t.List[t.List[str]]] = {}
69+
for plugin_type, plugins in plugin_info.items():
70+
for plugin_name, plugin_data in plugins.items():
71+
plugin_options: t.Mapping[str, t.Mapping[str, t.Any]] = (
72+
(plugin_data.get('doc') or {}).get('options') or {}
73+
)
74+
for env_var, env_var_description in _find_env_vars(plugin_options):
75+
if env_var in core_envs:
76+
continue
77+
if env_var not in other_variables:
78+
other_variables[env_var] = EnvironmentVariableInfo(env_var)
79+
other_variable_description[env_var] = []
80+
if plugin_type not in other_variables[env_var].plugins:
81+
other_variables[env_var].plugins[plugin_type] = []
82+
other_variables[env_var].plugins[plugin_type].append(plugin_name)
83+
if env_var_description is not None:
84+
other_variable_description[env_var].append(env_var_description)
85+
return other_variables, other_variable_description
86+
87+
88+
def _augment_env_var_descriptions(other_variables: t.Mapping[str, EnvironmentVariableInfo],
89+
other_variable_description: t.Mapping[str, t.List[t.List[str]]],
90+
) -> None:
91+
for variable, variable_info in other_variables.items():
92+
if other_variable_description[variable]:
93+
value: t.Optional[t.List[str]] = other_variable_description[variable][0]
94+
for other_value in other_variable_description[variable]:
95+
if value != other_value:
96+
value = [
97+
'See the documentations for the options where this environment variable'
98+
' is used.'
99+
]
100+
break
101+
variable_info.description = value
102+
103+
104+
def collect_referenced_environment_variables(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
105+
ansible_config: t.Mapping[str, t.Mapping[str, t.Any]],
106+
) -> t.Mapping[str, EnvironmentVariableInfo]:
107+
"""
108+
Collect referenced environment variables that are not defined in the ansible-core
109+
configuration.
110+
111+
:arg plugin_info: Mapping of plugin type to a mapping of plugin name to plugin record.
112+
:arg ansible_config: The Ansible base configuration (``lib/ansible/config/base.yml``).
113+
:returns: A Mapping of environment variable name to an environment variable infomation object.
114+
"""
115+
core_envs = {'ANSIBLE_CONFIG'}
116+
for config in ansible_config.values():
117+
if config.get('env'):
118+
for env in config['env']:
119+
core_envs.add(env['name'])
120+
121+
other_variables, other_variable_description = _collect_env_vars_and_descriptions(
122+
plugin_info, core_envs)
123+
_augment_env_var_descriptions(other_variables, other_variable_description)
124+
return other_variables

src/antsibull_docs/write_docs.py

+46-2
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@
2121

2222
from .jinja2.environment import doc_environment
2323
from .collection_links import CollectionLinks
24-
from .extra_docs import CollectionExtraDocsInfoT
2524
from .docs_parsing import AnsibleCollectionMetadata
25+
from .env_variables import EnvironmentVariableInfo
26+
from .extra_docs import CollectionExtraDocsInfoT
2627
from .utils.collection_name_transformer import CollectionNameTransformer
2728

2829

@@ -907,7 +908,7 @@ async def output_extra_docs(dest_dir: str,
907908
extra_docs_data: t.Mapping[str, CollectionExtraDocsInfoT],
908909
squash_hierarchy: bool = False) -> None:
909910
"""
910-
Generate collection-level index pages for the collections.
911+
Write extra docs pages for the collections.
911912
912913
:arg dest_dir: The directory to place the documentation in.
913914
:arg extra_docs_data: Dictionary mapping collection names to CollectionExtraDocsInfoT.
@@ -940,3 +941,46 @@ async def output_extra_docs(dest_dir: str,
940941
await asyncio.gather(*writers)
941942

942943
flog.debug('Leave')
944+
945+
946+
async def output_environment_variables(dest_dir: str,
947+
env_variables: t.Mapping[str, EnvironmentVariableInfo],
948+
squash_hierarchy: bool = False
949+
) -> None:
950+
"""
951+
Write environment variable Generate collection-level index pages for the collections.
952+
953+
:arg dest_dir: The directory to place the documentation in.
954+
:arg env_variables: Mapping of environment variable names to environment variable information.
955+
:arg squash_hierarchy: If set to ``True``, no directory hierarchy will be used.
956+
Undefined behavior if documentation for multiple collections are
957+
created.
958+
"""
959+
flog = mlog.fields(func='write_environment_variables')
960+
flog.debug('Enter')
961+
962+
if not squash_hierarchy:
963+
collection_toplevel = os.path.join(dest_dir, 'collections')
964+
else:
965+
collection_toplevel = dest_dir
966+
967+
env = doc_environment(('antsibull_docs.data', 'docsite'))
968+
# Get the templates
969+
env_var_list_tmpl = env.get_template('list_of_env_variables.rst.j2')
970+
971+
flog.fields(toplevel=collection_toplevel, exists=os.path.isdir(collection_toplevel)).debug(
972+
'collection_toplevel exists?')
973+
# This is only safe because we made sure that the top of the directory tree we're writing to
974+
# (docs/docsite/rst) is only writable by us.
975+
os.makedirs(collection_toplevel, mode=0o755, exist_ok=True)
976+
977+
index_file = os.path.join(collection_toplevel, 'environment_variables.rst')
978+
index_contents = _render_template(
979+
env_var_list_tmpl,
980+
index_file,
981+
env_variables=env_variables,
982+
)
983+
984+
await write_file(index_file, index_contents)
985+
986+
flog.debug('Leave')

0 commit comments

Comments
 (0)