Skip to content

Commit aea5933

Browse files
committed
Improve environment variable handling. Allow modules to declare environment variables as well, and create an environment variable index.
1 parent 06ccf9d commit aea5933

File tree

7 files changed

+249
-17
lines changed

7 files changed

+249
-17
lines changed

changelogs/fragments/73-env-vars.yml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
minor_changes:
2+
- "Use correct markup (``envvar`` role) for environment variables. Allow modules to declare environment variables. Compile an index of all environment variables used by plugins and modules. (https://github.com/ansible-community/antsibull-docs/pull/73)."

src/antsibull_docs/cli/doc_commands/stable.py

+21-4
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
load_all_collection_routing,
3838
remove_redirect_duplicates,
3939
)
40+
from ...env_variables import load_ansible_config, collect_referenced_environment_variables
4041
from ...schemas.docs import DOCS_SCHEMAS
4142
from ...utils.collection_name_transformer import CollectionNameTransformer
4243
from ...write_docs import (
@@ -47,6 +48,7 @@
4748
output_indexes,
4849
output_plugin_indexes,
4950
output_extra_docs,
51+
output_environment_variables,
5052
)
5153

5254
if t.TYPE_CHECKING:
@@ -337,12 +339,16 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
337339
app_ctx = app_context.app_ctx.get()
338340

339341
# Get the info from the plugins
340-
plugin_info, collection_metadata = asyncio_run(get_ansible_plugin_info(
342+
plugin_info, full_collection_metadata = asyncio_run(get_ansible_plugin_info(
341343
venv, collection_dir, collection_names=collection_names))
342344
flog.notice('Finished parsing info from plugins and collections')
343345
# flog.fields(plugin_info=plugin_info).debug('Plugin data')
344346
# flog.fields(
345-
# collection_metadata=collection_metadata).debug('Collection metadata')
347+
# collection_metadata=full_collection_metadata).debug('Collection metadata')
348+
349+
collection_metadata = dict(full_collection_metadata)
350+
if collection_names is not None and 'ansible.builtin' not in collection_names:
351+
del collection_metadata['ansible.builtin']
346352

347353
# Load collection routing information
348354
collection_routing = asyncio_run(load_all_collection_routing(collection_metadata))
@@ -363,7 +369,7 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
363369
{name: data.path for name, data in collection_metadata.items()}))
364370
flog.debug('Finished getting collection extra docs data')
365371

366-
# Load collection extra docs data
372+
# Load collection links data
367373
link_data = asyncio_run(load_collections_links(
368374
{name: data.path for name, data in collection_metadata.items()}))
369375
flog.debug('Finished getting collection link data')
@@ -384,6 +390,10 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
384390
print(f"{plugin_name} {plugin_type}: {textwrap.indent(error, ' ').lstrip()}")
385391
return 1
386392

393+
# Handle environment variables
394+
ansible_config = load_ansible_config(full_collection_metadata['ansible.builtin'])
395+
referenced_env_vars = collect_referenced_environment_variables(new_plugin_info, ansible_config)
396+
387397
collection_namespaces = get_collection_namespaces(collection_to_plugin_info.keys())
388398

389399
collection_url = CollectionNameTransformer(
@@ -444,7 +454,14 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
444454

445455
asyncio_run(output_extra_docs(dest_dir, extra_docs_data,
446456
squash_hierarchy=squash_hierarchy))
447-
flog.debug('Finished writing extra extra docs docs')
457+
flog.debug('Finished writing extra docs')
458+
459+
if referenced_env_vars:
460+
asyncio_run(output_environment_variables(dest_dir, referenced_env_vars,
461+
squash_hierarchy=squash_hierarchy))
462+
flog.debug('Finished writing environment variables')
463+
else:
464+
flog.debug('Skipping environment variables (as there are none)')
448465
return 0
449466

450467

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{#
2+
Copyright (c) Ansible Project
3+
GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
4+
SPDX-License-Identifier: GPL-3.0-or-later
5+
#}
6+
7+
:orphan:
8+
9+
.. _list_of_collection_env_vars:
10+
11+
Index of all Collection Environment Variables
12+
=============================================
13+
14+
The following index documents all environment variables declared by plugins and modules in collections.
15+
Environment variables used by the ansible-core configuation are documented in :ref:`ansible_configuration_settings`.
16+
{# TODO: use label `ansible_configuration_env_vars` once the ansible-core PR is merged #}
17+
18+
{% for _, env_var in env_variables | dictsort %}
19+
.. envvar:: @{ env_var.name }@
20+
21+
{% for paragraph in env_var.description or [] %}
22+
@{ paragraph | replace('\n', '\n ') | rst_ify | indent(4) }@
23+
24+
{% endfor %}
25+
*Used by:*
26+
{% set plugins_ = [] %}
27+
{% for plugin_type, plugins in env_var.plugins.items() %}
28+
{% for plugin_name in plugins %}
29+
{% set _ = plugins_.append((plugin_name, plugin_type)) %}
30+
{% endfor %}
31+
{% endfor %}
32+
{% for plugin_name, plugin_type in plugins_ | unique | sort %}
33+
:ref:`@{ plugin_name | rst_escape }@ {% if plugin_type == 'module' %}module{% else %}@{ plugin_type }@ plugin{% endif %} <ansible_collections.@{ plugin_name }@_@{ plugin_type }@>`
34+
{%- if not loop.last -%}
35+
,
36+
{% endif -%}
37+
{%- endfor %}
38+
39+
{% endfor %}

src/antsibull_docs/data/docsite/macros/parameters.rst.j2

+14-10
Original file line numberDiff line numberDiff line change
@@ -99,12 +99,16 @@
9999
:ansible-option-default-bold:`Default:` :ansible-option-default:`@{ value['default'] | antsibull_to_json | rst_escape(escape_ending_whitespace=true) | indent(6, blank=true) }@`
100100
{% endif %}
101101
{# Configuration #}
102-
{% if plugin_type != 'module' and plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}
102+
{% if plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}
103103

104104
.. rst-class:: ansible-option-line
105105

106106
:ansible-option-configuration:`Configuration:`
107107

108+
{% if plugin_type == 'module' and value['env'] %}
109+
The below environment variable{% if value['env'] | length > 1 %}s{% endif %} will be used on the host that executes this module.
110+
111+
{% endif %}
108112
{% if value['ini'] %}
109113
- INI {% if value['ini'] | length == 1 %}entry{% else %}entries{% endif %}:
110114
{% for ini in value['ini'] %}
@@ -125,30 +129,30 @@
125129
{% endfor %}
126130
{% endif %}
127131
{% for env in value['env'] %}
128-
- Environment variable: @{ env['name'] | rst_escape }@
132+
- Environment variable: :envvar:`@{ env['name'] | rst_escape(escape_ending_whitespace=true) }@`
129133
{% if env['version_added'] is still_relevant(collection=env['version_added_collection'] or collection) %}
130134

131135
:ansible-option-versionadded:`added in @{ version_added_rst(env['version_added'], env['version_added_collection'] or collection) }@`
132136
{% endif %}
133137
@{ deprecates_rst(env['deprecated'], collection, 8) }@
134138
{% endfor %}
135-
{% for myvar in value['vars'] %}
139+
{% for myvar in value['vars'] | default([]) %}
136140
- Variable: @{ myvar['name'] | rst_escape }@
137141
{% if myvar['version_added'] is still_relevant(collection=myvar['version_added_collection'] or collection) %}
138142

139143
:ansible-option-versionadded:`added in @{ version_added_rst(myvar['version_added'], myvar['version_added_collection'] or collection) }@`
140144
{% endif %}
141145
@{ deprecates_rst(myvar['deprecated'], collection, 8) }@
142146
{% endfor %}
143-
{% for kw in value['keyword'] %}
147+
{% for kw in value['keyword'] | default([]) %}
144148
- Keyword: @{ kw['name'] | rst_escape }@
145149
{% if kw['version_added'] is still_relevant(collection=kw['version_added_collection'] or collection) %}
146150

147151
:ansible-option-versionadded:`added in @{ version_added_rst(kw['version_added'], kw['version_added_collection'] or collection) }@`
148152
{% endif %}
149153
@{ deprecates_rst(kw['deprecated'], collection, 8) }@
150154
{% endfor %}
151-
{% for mycli in value['cli'] %}
155+
{% for mycli in value['cli'] | default([]) %}
152156
- CLI argument: @{ mycli['option'] | rst_escape }@
153157
{% if mycli['version_added'] is still_relevant(collection=mycli['version_added_collection'] or collection) %}
154158

@@ -228,7 +232,7 @@
228232
<p class="ansible-option-line"><span class="ansible-option-default-bold">Default:</span> <code class="ansible-value literal notranslate ansible-option-default">@{ value['default'] | antsibull_to_json | escape | indent(6, blank=true) }@</code></p>
229233
{% endif %}
230234
{# Configuration #}
231-
{% if plugin_type != 'module' and plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}
235+
{% if plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}
232236
<p class="ansible-option-line"><span class="ansible-option-configuration">Configuration:</span></p>
233237
<ul class="simple">
234238
{% if value['ini'] %}
@@ -250,14 +254,14 @@
250254
{% endif %}
251255
{% for env in value['env'] %}
252256
<li>
253-
<p>Environment variable: @{ env['name'] | escape }@</p>
257+
<p>Environment variable: <code class="xref std std-envvar literal notranslate">@{ env['name'] | escape }@</code></p>
254258
{% if env['version_added'] is still_relevant(collection=env['version_added_collection'] or collection) %}
255259
<p><span class="ansible-option-versionadded">added in @{ version_added_html(env['version_added'], env['version_added_collection'] or collection) }@</span></p>
256260
{% endif %}
257261
@{ deprecates_html(env['deprecated'], collection) }@
258262
</li>
259263
{% endfor %}
260-
{% for myvar in value['vars'] %}
264+
{% for myvar in value['vars'] | default([]) %}
261265
<li>
262266
<p>Variable: @{ myvar['name'] | escape }@</p>
263267
{% if myvar['version_added'] is still_relevant(collection=myvar['version_added_collection'] or collection) %}
@@ -266,7 +270,7 @@
266270
@{ deprecates_html(myvar['deprecated'], collection) }@
267271
</li>
268272
{% endfor %}
269-
{% for kw in value['keyword'] %}
273+
{% for kw in value['keyword'] | default([]) %}
270274
<li>
271275
<p>Keyword: @{ kw['name'] | escape }@</p>
272276
{% if kw['version_added'] is still_relevant(collection=kw['version_added_collection'] or collection) %}
@@ -275,7 +279,7 @@
275279
@{ deprecates_html(kw['deprecated'], collection) }@
276280
</li>
277281
{% endfor %}
278-
{% for mycli in value['cli'] %}
282+
{% for mycli in value['cli'] | default([]) %}
279283
<li>
280284
<p>CLI argument: @{ mycli['option'] | escape }@</p>
281285
{% if mycli['version_added'] is still_relevant(collection=mycli['version_added_collection'] or collection) %}

src/antsibull_docs/env_variables.py

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# Author: Felix Fontein <[email protected]>
2+
# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
3+
# https://www.gnu.org/licenses/gpl-3.0.txt)
4+
# SPDX-License-Identifier: GPL-3.0-or-later
5+
# SPDX-FileCopyrightText: 2022, Ansible Project
6+
"""Environment variable handling."""
7+
8+
import os
9+
import os.path
10+
import typing as t
11+
12+
from antsibull_core import yaml
13+
14+
from .docs_parsing import AnsibleCollectionMetadata
15+
16+
17+
class EnvironmentVariableInfo:
18+
name: str
19+
description: t.Optional[t.List[str]]
20+
plugins: t.Dict[str, t.List[str]] # maps plugin_type to lists of plugin FQCNs
21+
22+
def __init__(self,
23+
name: str,
24+
description: t.Optional[t.List[str]] = None,
25+
plugins: t.Optional[t.Dict[str, t.List[str]]] = None):
26+
self.name = name
27+
self.description = description
28+
self.plugins = plugins or {}
29+
30+
def __repr__(self):
31+
return f'E({self.name}, description={repr(self.description)}, plugins={self.plugins})'
32+
33+
34+
def load_ansible_config(ansible_builtin_metadata: AnsibleCollectionMetadata
35+
) -> t.Mapping[str, t.Mapping[str, t.Any]]:
36+
"""
37+
Load Ansible base configuration (``lib/ansible/config/base.yml``).
38+
39+
:arg ansible_builtin_metadata: Metadata for the ansible.builtin collection.
40+
:returns: A Mapping of configuration options to information on these options.
41+
"""
42+
return yaml.load_yaml_file(os.path.join(ansible_builtin_metadata.path, 'config', 'base.yml'))
43+
44+
45+
def _find_env_vars(options: t.Mapping[str, t.Mapping[str, t.Any]]
46+
) -> t.Generator[t.Tuple[str, t.Optional[t.List[str]]], None, None]:
47+
for _, option_data in options.items():
48+
if isinstance(option_data.get('env'), list):
49+
description = option_data.get('description')
50+
if isinstance(description, str):
51+
description = [description]
52+
if isinstance(description, list):
53+
description = [str(desc) for desc in description]
54+
else:
55+
description = None
56+
for env_var in option_data['env']:
57+
if isinstance(env_var.get('name'), str):
58+
yield (env_var['name'], description)
59+
if isinstance(option_data.get('suboptions'), dict):
60+
yield from _find_env_vars(option_data['suboptions'])
61+
62+
63+
def _collect_env_vars_and_descriptions(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
64+
core_envs: t.Set[str],
65+
) -> t.Tuple[t.Mapping[str, EnvironmentVariableInfo],
66+
t.Mapping[str, t.List[t.List[str]]]]:
67+
other_variables: t.Dict[str, EnvironmentVariableInfo] = {}
68+
other_variable_description: t.Dict[str, t.List[t.List[str]]] = {}
69+
for plugin_type, plugins in plugin_info.items():
70+
for plugin_name, plugin_data in plugins.items():
71+
plugin_options: t.Mapping[str, t.Mapping[str, t.Any]] = (
72+
(plugin_data.get('doc') or {}).get('options') or {}
73+
)
74+
for env_var, env_var_description in _find_env_vars(plugin_options):
75+
if env_var in core_envs:
76+
continue
77+
if env_var not in other_variables:
78+
other_variables[env_var] = EnvironmentVariableInfo(env_var)
79+
other_variable_description[env_var] = []
80+
if plugin_type not in other_variables[env_var].plugins:
81+
other_variables[env_var].plugins[plugin_type] = []
82+
other_variables[env_var].plugins[plugin_type].append(plugin_name)
83+
if env_var_description is not None:
84+
other_variable_description[env_var].append(env_var_description)
85+
return other_variables, other_variable_description
86+
87+
88+
def _augment_env_var_descriptions(other_variables: t.Mapping[str, EnvironmentVariableInfo],
89+
other_variable_description: t.Mapping[str, t.List[t.List[str]]],
90+
) -> None:
91+
for variable, variable_info in other_variables.items():
92+
if other_variable_description[variable]:
93+
value: t.Optional[t.List[str]] = other_variable_description[variable][0]
94+
for other_value in other_variable_description[variable]:
95+
if value != other_value:
96+
value = [
97+
'See the documentations for the options where this environment variable'
98+
' is used.'
99+
]
100+
break
101+
variable_info.description = value
102+
103+
104+
def collect_referenced_environment_variables(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
105+
ansible_config: t.Mapping[str, t.Mapping[str, t.Any]],
106+
) -> t.Mapping[str, EnvironmentVariableInfo]:
107+
"""
108+
Collect referenced environment variables that are not defined in the ansible-core
109+
configuration.
110+
111+
:arg plugin_info: Mapping of plugin type to a mapping of plugin name to plugin record.
112+
:arg ansible_config: The Ansible base configuration (``lib/ansible/config/base.yml``).
113+
:returns: A Mapping of environment variable name to an environment variable infomation object.
114+
"""
115+
core_envs = {'ANSIBLE_CONFIG'}
116+
for config in ansible_config.values():
117+
if config.get('env'):
118+
for env in config['env']:
119+
core_envs.add(env['name'])
120+
121+
other_variables, other_variable_description = _collect_env_vars_and_descriptions(
122+
plugin_info, core_envs)
123+
_augment_env_var_descriptions(other_variables, other_variable_description)
124+
return other_variables

src/antsibull_docs/schemas/docs/module.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,11 @@
1010
import pydantic as p
1111

1212
from .base import BaseModel, DocSchema, OptionsSchema
13-
from .plugin import PluginExamplesSchema, PluginMetadataSchema, PluginReturnSchema
13+
from .plugin import OptionEnvSchema, PluginExamplesSchema, PluginMetadataSchema, PluginReturnSchema
1414

1515

1616
class InnerModuleOptionsSchema(OptionsSchema):
17+
env: t.List[OptionEnvSchema] = []
1718
suboptions: t.Dict[str, 'InnerModuleOptionsSchema'] = {}
1819

1920
@p.root_validator(pre=True)
@@ -29,6 +30,7 @@ def allow_description_to_be_optional(cls, values):
2930

3031

3132
class ModuleOptionsSchema(OptionsSchema):
33+
env: t.List[OptionEnvSchema] = []
3234
suboptions: t.Dict[str, 'InnerModuleOptionsSchema'] = {}
3335

3436

0 commit comments

Comments
 (0)