Skip to content

Commit 5bb7bc5

Browse files
felixfonteinfelixfontein
authored
Remove sh usages (#122)
* Began removing sh usages. * Drop antsibull-core 1.x.y support. Drop direct sh dependency. * Fix return type. Remove unnecessary imports and ignores. * Import CalledProcessError from antsibull_core.subprocess_util. * Document that we no longer depend directly on sh. * p.stdout is already a string.
1 parent 4f9887f commit 5bb7bc5

File tree

9 files changed

+63
-74
lines changed

9 files changed

+63
-74
lines changed

‎.github/workflows/antsibull-docs.yml

-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ jobs:
3838
include:
3939
- options: '--use-current --use-html-blobs --no-breadcrumbs community.crypto community.docker'
4040
python: '3.9'
41-
antsibull_core_ref: stable-1
4241

4342
steps:
4443
- name: Check out antsibull-docs

‎changelogs/fragments/122-antsibull-core-sh.yml

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
minor_changes:
2+
- "Now depends antsibull-core 2.0.0 or newer; antsibull-core 1.x.y is no longer supported (https://github.com/ansible-community/antsibull-docs/pull/122)."
3+
- "Ansibull-docs now no longer depends directly on ``sh`` (https://github.com/ansible-community/antsibull-docs/pull/122)."

‎pyproject.toml

+2-8
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,15 @@ classifiers = [
2525
requires-python = ">=3.9"
2626
dependencies = [
2727
"ansible-pygments",
28-
"antsibull-core >= 1.2.0, < 3.0.0",
28+
# TODO: bump to >= 2.0.0
29+
"antsibull-core >= 2.0.0a1, < 3.0.0",
2930
"antsibull-docs-parser ~= 0.3.0",
3031
"asyncio-pool",
3132
"docutils",
3233
"jinja2 >= 3.0",
3334
"packaging",
3435
"rstcheck >= 3.0.0, < 7.0.0",
3536
"sphinx",
36-
# sh v2 has breaking changes.
37-
# https://github.com/ansible-community/antsibull-core/issues/34
38-
"sh >= 1.0.0, < 2.0.0",
3937
# pydantic v2 is a major rewrite
4038
"pydantic >= 1.0.0, < 2.0.0",
4139
"semantic_version",
@@ -124,10 +122,6 @@ source = [
124122
[tool.mypy]
125123
mypy_path = "stubs/"
126124

127-
[[tool.mypy.overrides]]
128-
module = "sh"
129-
ignore_missing_imports = true
130-
131125
[[tool.mypy.overrides]]
132126
module = "semantic_version"
133127
ignore_missing_imports = true

‎src/antsibull_docs/cli/doc_commands/plugin.py

+7-11
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@
1212
import traceback
1313
import typing as t
1414

15-
import sh
1615
from antsibull_core.compat import asyncio_run
1716
from antsibull_core.logging import log
17+
from antsibull_core.subprocess_util import CalledProcessError
1818
from antsibull_core.vendored.json_utils import _filter_non_json_lines
1919
from antsibull_core.venv import FakeVenvRunner
2020

@@ -47,26 +47,22 @@ def generate_plugin_docs(plugin_type: str, plugin_name: str,
4747
venv_ansible_doc = venv.get_command('ansible-doc')
4848
venv_ansible_doc = venv_ansible_doc.bake('-vvv')
4949
try:
50-
ansible_doc_results = venv_ansible_doc('-t', plugin_type, '--json', plugin_name)
51-
except sh.ErrorReturnCode as exc:
50+
ansible_doc_results = venv.log_run(
51+
['ansible-doc', '-vvv', '-t', plugin_type, '--json', plugin_name])
52+
except CalledProcessError as exc:
5253
err_msg = []
5354
formatted_exception = traceback.format_exception(None, exc, exc.__traceback__)
5455
err_msg.append(f'Exception while parsing documentation for {plugin_type} plugin:'
5556
f' {plugin_name}. Will not document this plugin.')
5657
err_msg.append(f'Exception:\n{"".join(formatted_exception)}')
5758

58-
stdout = exc.stdout.decode("utf-8", errors="surrogateescape")
59-
stderr = exc.stderr.decode("utf-8", errors="surrogateescape")
60-
61-
err_msg.append(f'Full process stdout:\n{stdout}')
62-
err_msg.append(f'Full process stderr:\n{stderr}')
59+
err_msg.append(f'Full process stdout:\n{exc.stdout}')
60+
err_msg.append(f'Full process stderr:\n{exc.stderr}')
6361

6462
sys.stderr.write('\n'.join(err_msg))
6563
return 1
6664

67-
stdout = ansible_doc_results.stdout.decode("utf-8", errors="surrogateescape")
68-
69-
plugin_data = json.loads(_filter_non_json_lines(stdout)[0])
65+
plugin_data = json.loads(_filter_non_json_lines(ansible_doc_results.stdout)[0])
7066
try:
7167
plugin_info = plugin_data[plugin_name]
7268
except KeyError:

‎src/antsibull_docs/docs_parsing/ansible_doc.py

+29-32
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
import re
1111
import typing as t
1212

13-
import sh
1413
from antsibull_core.logging import log
14+
from antsibull_core.subprocess_util import CalledProcessError
1515
from antsibull_core.vendored.json_utils import _filter_non_json_lines
1616
from packaging.version import Version as PypiVer
1717

@@ -64,38 +64,37 @@ def parse_ansible_galaxy_collection_list(json_output: t.Mapping[str, t.Any],
6464
return result
6565

6666

67-
def _call_ansible_version(
67+
async def _call_ansible_version(
6868
venv: t.Union['VenvRunner', 'FakeVenvRunner'],
69-
env: t.Dict[str, str],
69+
env: t.Optional[t.Dict[str, str]],
7070
) -> str:
71-
venv_ansible = venv.get_command('ansible')
72-
ansible_version_cmd = venv_ansible('--version', _env=env)
73-
return ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape')
71+
p = await venv.async_log_run(['ansible', '--version'], env=env)
72+
return p.stdout
7473

7574

76-
def _call_ansible_galaxy_collection_list(
75+
async def _call_ansible_galaxy_collection_list(
7776
venv: t.Union['VenvRunner', 'FakeVenvRunner'],
7877
env: t.Dict[str, str],
7978
) -> t.Mapping[str, t.Any]:
80-
venv_ansible_galaxy = venv.get_command('ansible-galaxy')
81-
ansible_collection_list_cmd = venv_ansible_galaxy(
82-
'collection', 'list', '--format', 'json', _env=env)
83-
stdout = ansible_collection_list_cmd.stdout.decode('utf-8', errors='surrogateescape')
84-
return json.loads(_filter_non_json_lines(stdout)[0])
79+
p = await venv.async_log_run(
80+
['ansible-galaxy', 'collection', 'list', '--format', 'json'],
81+
env=env,
82+
)
83+
return json.loads(_filter_non_json_lines(p.stdout)[0])
8584

8685

87-
def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
88-
env: t.Dict[str, str],
89-
collection_names: t.Optional[t.List[str]] = None,
90-
) -> t.Dict[str, AnsibleCollectionMetadata]:
86+
async def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
87+
env: t.Dict[str, str],
88+
collection_names: t.Optional[t.List[str]] = None,
89+
) -> t.Dict[str, AnsibleCollectionMetadata]:
9190
collection_metadata = {}
9291

9392
# Obtain ansible.builtin version and path
94-
raw_result = _call_ansible_version(venv, env)
93+
raw_result = await _call_ansible_version(venv, env)
9594
collection_metadata['ansible.builtin'] = _extract_ansible_builtin_metadata(raw_result)
9695

9796
# Obtain collection versions
98-
json_result = _call_ansible_galaxy_collection_list(venv, env)
97+
json_result = await _call_ansible_galaxy_collection_list(venv, env)
9998
collection_list = parse_ansible_galaxy_collection_list(json_result, collection_names)
10099
for namespace, name, path, version in collection_list:
101100
collection_name = f'{namespace}.{name}'
@@ -105,28 +104,26 @@ def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
105104
return collection_metadata
106105

107106

108-
def get_ansible_core_version(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
109-
env: t.Optional[t.Dict[str, str]] = None,
110-
) -> PypiVer:
111-
try:
112-
venv_python = venv.get_command('python')
113-
ansible_version_cmd = venv_python(
114-
'-c', 'import ansible.release; print(ansible.release.__version__)', _env=env)
115-
output = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape').strip()
107+
async def get_ansible_core_version(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
108+
env: t.Optional[t.Dict[str, str]] = None,
109+
) -> PypiVer:
110+
p = await venv.async_log_run(
111+
['python', '-c', 'import ansible.release; print(ansible.release.__version__)'],
112+
env=env,
113+
check=False,
114+
)
115+
output = p.stdout.strip()
116+
if p.returncode == 0 and output:
116117
return PypiVer(output)
117-
except sh.ErrorReturnCode:
118-
pass
119118

120119
try:
121120
# Fallback: use `ansible --version`
122-
venv_ansible = venv.get_command('ansible')
123-
ansible_version_cmd = venv_ansible('--version', _env=env)
124-
raw_result = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape')
121+
raw_result = await _call_ansible_version(venv, env)
125122
metadata = _extract_ansible_builtin_metadata(raw_result)
126123
if metadata.version is None:
127124
raise ValueError('Cannot retrieve ansible-core version from `ansible --version`')
128125
return PypiVer(metadata.version)
129-
except sh.ErrorReturnCode as exc:
126+
except CalledProcessError as exc:
130127
raise ValueError(
131128
f'Cannot retrieve ansible-core version from `ansible --version`: {exc}'
132129
) from exc

‎src/antsibull_docs/docs_parsing/ansible_doc_core_213.py

+9-11
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,16 @@
2323
mlog = log.fields(mod=__name__)
2424

2525

26-
def _call_ansible_doc(
26+
async def _call_ansible_doc(
2727
venv: t.Union['VenvRunner', 'FakeVenvRunner'],
2828
env: t.Dict[str, str],
2929
*parameters: str,
3030
) -> t.Mapping[str, t.Any]:
31-
# Setup an sh.Command to run ansible-doc from the venv with only the collections we
32-
# found as providers of extra plugins.
33-
venv_ansible_doc = venv.get_command('ansible-doc')
34-
venv_ansible_doc = venv_ansible_doc.bake('-vvv', _env=env)
35-
ansible_doc_call = venv_ansible_doc('--metadata-dump', '--no-fail-on-errors', *parameters)
36-
stdout = ansible_doc_call.stdout.decode('utf-8', errors='surrogateescape')
37-
return json.loads(_filter_non_json_lines(stdout)[0])
31+
p = await venv.async_log_run(
32+
['ansible-doc', '-vvv', '--metadata-dump', '--no-fail-on-errors', *parameters],
33+
env=env,
34+
)
35+
return json.loads(_filter_non_json_lines(p.stdout)[0])
3836

3937

4038
async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
@@ -69,9 +67,9 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
6967
flog.debug('Retrieving and loading plugin documentation')
7068
if collection_names and len(collection_names) == 1:
7169
# ansible-doc only allows *one* filter
72-
ansible_doc_output = _call_ansible_doc(venv, env, collection_names[0])
70+
ansible_doc_output = await _call_ansible_doc(venv, env, collection_names[0])
7371
else:
74-
ansible_doc_output = _call_ansible_doc(venv, env)
72+
ansible_doc_output = await _call_ansible_doc(venv, env)
7573

7674
flog.debug('Processing plugin documentation')
7775
plugin_map: t.MutableMapping[str, t.MutableMapping[str, t.Any]] = {}
@@ -120,7 +118,7 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
120118
plugin_type_data[fqcn] = plugin_data
121119

122120
flog.debug('Retrieving collection metadata')
123-
collection_metadata = get_collection_metadata(venv, env, collection_names)
121+
collection_metadata = await get_collection_metadata(venv, env, collection_names)
124122

125123
flog.debug('Leave')
126124
return (plugin_map, collection_metadata)

‎src/antsibull_docs/docs_parsing/parsing.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
5555

5656
doc_parsing_backend = app_ctx.doc_parsing_backend
5757
if doc_parsing_backend == 'auto':
58-
version = get_ansible_core_version(venv)
58+
version = await get_ansible_core_version(venv)
5959
flog.debug(f'Ansible-core version: {version}')
6060
if version < PypiVer('2.13.0.dev0'):
6161
raise RuntimeError(f'Unsupported ansible-core version {version}. Need 2.13.0 or later.')

‎src/antsibull_docs/lint_plugin_docs.py

+3-4
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@
1212
import typing as t
1313
from collections.abc import Sequence
1414

15-
import sh
1615
from antsibull_core.compat import asyncio_run
16+
from antsibull_core.subprocess_util import log_run
1717
from antsibull_core.vendored.json_utils import _filter_non_json_lines
1818
from antsibull_core.venv import FakeVenvRunner
1919

@@ -74,9 +74,8 @@ def __exit__(self, type_, value, traceback_):
7474
class CollectionFinder:
7575
def __init__(self):
7676
self.collections = {}
77-
stdout = sh.Command('ansible-galaxy')('collection', 'list', '--format', 'json').stdout
78-
raw_output = stdout.decode('utf-8', errors='surrogateescape')
79-
data = json.loads(_filter_non_json_lines(raw_output)[0])
77+
p = log_run(['ansible-galaxy', 'collection', 'list', '--format', 'json'])
78+
data = json.loads(_filter_non_json_lines(p.stdout)[0])
8079
for namespace, name, path, _ in reversed(parse_ansible_galaxy_collection_list(data)):
8180
self.collections[f'{namespace}.{name}'] = path
8281

‎tests/functional/ansible_doc_caching.py

+9-6
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
@contextmanager
1818
def ansible_doc_cache():
19-
def call_ansible_doc(
19+
async def call_ansible_doc(
2020
venv: t.Union['VenvRunner', 'FakeVenvRunner'],
2121
env: t.Dict[str, str],
2222
*parameters: str,
@@ -45,19 +45,22 @@ def call_ansible_doc(
4545
doc[key] = os.path.join(root, doc[key])
4646
return data
4747

48-
def call_ansible_version(
48+
async def call_ansible_version(
4949
venv: t.Union['VenvRunner', 'FakeVenvRunner'],
50-
env: t.Dict[str, str],
50+
env: t.Optional[t.Dict[str, str]],
5151
) -> str:
5252
filename = os.path.join(os.path.dirname(__file__), 'ansible-version.output')
5353
with open(filename, 'rt', encoding='utf-8') as f:
5454
content = f.read()
5555

56-
root = env['ANSIBLE_COLLECTIONS_PATH']
57-
return content.replace('<<<<<COLLECTIONS>>>>>', root).replace('<<<<<HOME>>>>>', env['HOME']).replace('<<<<<ANSIBLE>>>>>', os.path.dirname(ansible.__file__))
56+
root = env['ANSIBLE_COLLECTIONS_PATH'] if env and 'ANSIBLE_COLLECTIONS_PATH' in env else '/collections'
57+
content = content.replace('<<<<<COLLECTIONS>>>>>', root)
58+
content = content.replace('<<<<<HOME>>>>>', (env or os.environ)['HOME'])
59+
content = content.replace('<<<<<ANSIBLE>>>>>', os.path.dirname(ansible.__file__))
60+
return content
5861

5962

60-
def call_ansible_galaxy_collection_list(
63+
async def call_ansible_galaxy_collection_list(
6164
venv: t.Union['VenvRunner', 'FakeVenvRunner'],
6265
env: t.Dict[str, str],
6366
) -> t.Mapping[str, t.Any]:

0 commit comments

Comments
 (0)