Add baseline tests (#50)

* Start with baseline tests.

* Run baseline tests only with new enough Python and when ansible-core is installed in venv.

* Improve ansible --version fallback.

* Add changelog fragment for code change.

* Add blanket license to baseline files.

This is OK since they are generated by code from this repository
from files inside this repository.

* Extend baseline tests.

* Make baseline build script nicer.

* Make sure that at least for one Python version no tests are skipped.

Author: felixfontein

.github/workflows/pythontests.yml

@@ -23,7 +23,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, '3.10']
+        python-version: [3.6, 3.7, 3.8, 3.9]
+        skip-is-error: [false]
+        include:
+          - python-version: '3.10'
+            skip-is-error: true
 
     steps:
       - name: Check out antsibull-docs
@@ -56,7 +60,7 @@ jobs:
 
       - name: Test with pytest and upload coverage stats
         run: |
-          ./test-pytest.sh
+          ./test-pytest.sh ${{ matrix.skip-is-error && '--error-for-skips' || '' }}
           poetry run coverage xml -i
           poetry run codecov
         working-directory: antsibull-docs

.reuse/dep5

@@ -3,3 +3,7 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Files: changelogs/fragments/*
 Copyright: Ansible Project
 License: GPL-3.0-or-later
+
+Files: tests/functional/baseline-*
+Copyright: Ansible Project
+License: GPL-3.0-or-later

changelogs/fragments/50-ansible-version-fallback.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - "Use ``ansible --version`` to figure out ansible-core version when ansible-core is not installed for the same Python interpreter / venv that is used for antsibull-docs (https://github.com/ansible-community/antsibull-docs/pull/50)."

pyproject.toml

@@ -43,6 +43,7 @@ rstcheck = ">= 3.0.0, < 7.0.0"
 sphinx = "*"
 
 [tool.poetry.dev-dependencies]
+ansible-core = {version = ">= 2.14.0b1", python = ">=3.9"}
 asynctest = "*"
 cryptography = "*"
 codecov = "*"
@@ -54,6 +55,7 @@ pylint = "^2.12.0"
 pytest = "*"
 pytest-asyncio = ">= 0.12"
 pytest-cov = "*"
+pytest-error-for-skips = "*"
 # Needed for TypedDict in rstcheck-core stubs
 typing-extensions = {version = ">=3.7.4", python = "<3.8"}
 types-docutils = "*"

src/antsibull_docs/docs_parsing/ansible_doc.py

@@ -239,11 +239,28 @@ def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
 def get_ansible_core_version(venv: t.Union['VenvRunner', 'FakeVenvRunner'],
                              env: t.Optional[t.Dict[str, str]] = None,
                              ) -> PypiVer:
-    venv_python = venv.get_command('python')
-    ansible_version_cmd = venv_python(
-        '-c', 'import ansible.release; print(ansible.release.__version__)', _env=env)
-    output = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape').strip()
-    return PypiVer(output)
+    try:
+        venv_python = venv.get_command('python')
+        ansible_version_cmd = venv_python(
+            '-c', 'import ansible.release; print(ansible.release.__version__)', _env=env)
+        output = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape').strip()
+        return PypiVer(output)
+    except sh.ErrorReturnCode:
+        pass
+
+    try:
+        # Fallback: use `ansible --version`
+        venv_ansible = venv.get_command('ansible')
+        ansible_version_cmd = venv_ansible('--version', _env=env)
+        raw_result = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape')
+        metadata = _extract_ansible_builtin_metadata(raw_result)
+        if metadata.version is None:
+            raise ValueError('Cannot retrieve ansible-core version from `ansible --version`')
+        return PypiVer(metadata.version)
+    except sh.ErrorReturnCode as exc:
+        raise ValueError(
+            f'Cannot retrieve ansible-core version from `ansible --version`: {exc}'
+        ) from exc
 
 
 async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'],

tests/functional/baseline-default/collections/index.rst

@@ -0,0 +1,20 @@
+
+:orphan:
+
+.. _list_of_collections:
+
+Collection Index
+================
+
+These are the collections documented here.
+
+* :ref:`ns.col1 <plugins_in_ns.col1>`
+* :ref:`ns.col2 <plugins_in_ns.col2>`
+* :ref:`ns2.col <plugins_in_ns2.col>`
+
+.. toctree::
+    :maxdepth: 1
+    :hidden:
+
+    ns/index
+    ns2/index

tests/functional/baseline-default/collections/index_become.rst

@@ -0,0 +1,13 @@
+
+:orphan:
+
+.. _list_of_become_plugins:
+
+Index of all Become Plugins
+===========================
+
+ns2.col
+-------
+
+* :ref:`ns2.col.foo <ansible_collections.ns2.col.foo_become>` -- Use foo
+

tests/functional/baseline-default/collections/index_cache.rst

@@ -0,0 +1,13 @@
+
+:orphan:
+
+.. _list_of_cache_plugins:
+
+Index of all Cache Plugins
+==========================
+
+ns2.col
+-------
+
+* :ref:`ns2.col.foo <ansible_collections.ns2.col.foo_cache>` -- Foo files
+

tests/functional/baseline-default/collections/index_callback.rst

@@ -0,0 +1,13 @@
+
+:orphan:
+
+.. _list_of_callback_plugins:
+
+Index of all Callback Plugins
+=============================
+
+ns2.col
+-------
+
+* :ref:`ns2.col.foo <ansible_collections.ns2.col.foo_callback>` -- Foo output
+

tests/functional/baseline-default/collections/index_cliconf.rst

@@ -0,0 +1,13 @@
+
+:orphan:
+
+.. _list_of_cliconf_plugins:
+
+Index of all Cliconf Plugins
+============================
+
+ns2.col
+-------
+
+* :ref:`ns2.col.foo <ansible_collections.ns2.col.foo_cliconf>` -- Foo router CLI config
+