Skip to content

Commit 878eebb

Browse files
authored
Improve formatting, make sure unformatted text is properly escaped (#22)
* Add basic parser. * Use new parser to transform formatted blocks to RST resp. HTML. * Add changelog fragment. * Make work with Python 3.6. * Format error messages instead of raising exceptions (which break the build). * Also indent blank lines in HTML to avoid broken RST (when embedding HTML). * Linting.
1 parent ab23936 commit 878eebb

File tree

11 files changed

+757
-132
lines changed

11 files changed

+757
-132
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
minor_changes:
2+
- "When processing formatting directives, make sure to properly escape all other text for RST respectively HTML instead of including it verbatim (https://github.com/ansible-community/antsibull-docs/issues/21, https://github.com/ansible-community/antsibull-docs/pull/22)."
3+
bugfixes:
4+
- "Improve indentation of HTML blocks for tables to avoid edge cases which generate invalid RST (https://github.com/ansible-community/antsibull-docs/pull/22)."

src/antsibull_docs/data/docsite/macros/deprecates.rst.j2

+2-2
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ Removed in: a future release
4242
of @{ data['removed_from_collection'] | html_ify }@
4343
{%- endif -%}
4444
</p>
45-
<p>@{ 'Why: ' ~ data['why'] | html_ify | indent(2) }@</p>
46-
<p>@{ 'Alternative: ' ~ data['alternative'] | html_ify | indent(2) }@</p>
45+
<p>@{ 'Why: ' ~ data['why'] | html_ify | indent(2, blank=true) }@</p>
46+
<p>@{ 'Alternative: ' ~ data['alternative'] | html_ify | indent(2, blank=true) }@</p>
4747
{% endif %}
4848
{% endmacro %}

src/antsibull_docs/data/docsite/macros/parameters.rst.j2

+3-3
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@
111111

112112
.. rst-class:: ansible-option-line
113113

114-
:ansible-option-default-bold:`Default:` :ansible-option-default:`@{ value['default'] | antsibull_to_json | rst_escape(escape_ending_whitespace=true) | indent(6) }@`
114+
:ansible-option-default-bold:`Default:` :ansible-option-default:`@{ value['default'] | antsibull_to_json | rst_escape(escape_ending_whitespace=true) | indent(6, blank=true) }@`
115115
{% endif %}
116116
{# Configuration #}
117117
{% if plugin_type != 'module' and plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}
@@ -223,7 +223,7 @@
223223
{# description #}
224224
<td>{% for i in range(1, loop.depth) %}<div class="ansible-option-indent-desc"></div>{% endfor %}<div class="ansible-option-cell">
225225
{% for desc in value['description'] %}
226-
<p>@{ desc | html_ify | indent(6) }@</p>
226+
<p>@{ desc | html_ify | indent(6, blank=true) }@</p>
227227
{% endfor %}
228228
{# default / choices #}
229229
{# Turn boolean values in 'yes' and 'no' values #}
@@ -256,7 +256,7 @@
256256
{% endif %}
257257
{# Show default value, when multiple choice or no choices #}
258258
{% if value['default'] is not none and value['default'] not in value['choices'] %}
259-
<p class="ansible-option-line"><span class="ansible-option-default-bold">Default:</span> <span class="ansible-option-default">@{ value['default'] | antsibull_to_json | escape | indent(6) }@</span></p>
259+
<p class="ansible-option-line"><span class="ansible-option-default-bold">Default:</span> <span class="ansible-option-default">@{ value['default'] | antsibull_to_json | escape | indent(6, blank=true) }@</span></p>
260260
{% endif %}
261261
{# Configuration #}
262262
{% if plugin_type != 'module' and plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}

src/antsibull_docs/data/docsite/macros/returnvalues.rst.j2

+3-3
Original file line numberDiff line numberDiff line change
@@ -142,10 +142,10 @@
142142
{# description #}
143143
<td>{% for i in range(1, loop.depth) %}<div class="ansible-option-indent-desc"></div>{% endfor %}<div class="ansible-option-cell">
144144
{% for desc in value['description'] %}
145-
<p>@{ desc | html_ify | indent(6) }@</p>
145+
<p>@{ desc | html_ify | indent(6, blank=true) }@</p>
146146
{% endfor %}
147147
{% if value['returned'] %}
148-
<p class="ansible-option-line"><span class="ansible-option-returned-bold">Returned:</span> @{ value['returned'] | html_ify | indent(6) }@</p>
148+
<p class="ansible-option-line"><span class="ansible-option-returned-bold">Returned:</span> @{ value['returned'] | html_ify | indent(6, blank=true) }@</p>
149149
{% endif %}
150150
{# Show possible choices and highlight details #}
151151
{% if value['choices'] %}
@@ -167,7 +167,7 @@
167167
</ul>
168168
{% endif %}
169169
{% if value['sample'] is not none %}
170-
<p class="ansible-option-line ansible-option-sample"><span class="ansible-option-sample-bold">Sample:</span> @{ value['sample'] | antsibull_to_json | escape | indent(6) }@</p>
170+
<p class="ansible-option-line ansible-option-sample"><span class="ansible-option-sample-bold">Sample:</span> @{ value['sample'] | antsibull_to_json | escape | indent(6, blank=true) }@</p>
171171
{% endif %}
172172
</div></td>
173173
</tr>

src/antsibull_docs/jinja2/environment.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@
1010
from jinja2 import Environment, FileSystemLoader, PackageLoader
1111

1212
from .filters import (
13-
do_max, documented_type, html_ify, rst_ify, rst_escape, rst_fmt, rst_xline, move_first,
14-
massage_author_name, extract_options_from_list, remove_options_from_list, to_json,
13+
do_max, documented_type, rst_fmt, rst_xline, move_first, massage_author_name,
14+
extract_options_from_list, remove_options_from_list, to_json,
1515
)
16+
from .htmlify import html_ify
17+
from .rstify import rst_ify, rst_escape
1618
from .tests import still_relevant, test_list
1719

1820

src/antsibull_docs/jinja2/filters.py

-121
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88

99
import json
1010
import re
11-
from html import escape as html_escape
12-
from urllib.parse import quote
1311

1412
import typing as t
1513

@@ -20,45 +18,9 @@
2018

2119
mlog = log.fields(mod=__name__)
2220

23-
# Warning: If you add to this, then you also have to change ansible-doc
24-
# (ansible/cli/__init__.py) in the ansible/ansible repository
25-
_ITALIC = re.compile(r"\bI\(([^)]+)\)")
26-
_BOLD = re.compile(r"\bB\(([^)]+)\)")
27-
_MODULE = re.compile(r"\bM\(([^).]+)\.([^).]+)\.([^)]+)\)")
28-
_URL = re.compile(r"\bU\(([^)]+)\)")
29-
_LINK = re.compile(r"\bL\(([^)]+), *([^)]+)\)")
30-
_REF = re.compile(r"\bR\(([^)]+), *([^)]+)\)")
31-
_CONST = re.compile(r"\bC\(([^)]+)\)")
32-
_RULER = re.compile(r"\bHORIZONTALLINE\b")
33-
3421
_EMAIL_ADDRESS = re.compile(r"(?:<{mail}>|\({mail}\)|{mail})".format(mail=r"[\w.+-]+@[\w.-]+\.\w+"))
3522

3623

37-
def html_ify(text):
38-
''' convert symbols like I(this is in italics) to valid HTML '''
39-
40-
flog = mlog.fields(func='html_ify')
41-
flog.fields(text=text).debug('Enter')
42-
_counts = {}
43-
44-
text = html_escape(text)
45-
text, _counts['italic'] = _ITALIC.subn(r"<em>\1</em>", text)
46-
text, _counts['bold'] = _BOLD.subn(r"<b>\1</b>", text)
47-
text, _counts['module'] = _MODULE.subn(
48-
r"<a href='../../\1/\2/\3_module.html' class='module'>\1.\2.\3</a>", text)
49-
text, _counts['url'] = _URL.subn(r"<a href='\1'>\1</a>", text)
50-
text, _counts['ref'] = _REF.subn(r"<span class='module'>\1</span>", text)
51-
text, _counts['link'] = _LINK.subn(r"<a href='\2'>\1</a>", text)
52-
text, _counts['const'] = _CONST.subn(
53-
r"<code class='docutils literal notranslate'>\1</code>", text)
54-
text, _counts['ruler'] = _RULER.subn(r"<hr/>", text)
55-
56-
text = text.strip()
57-
flog.fields(counts=_counts).info('Number of macros converted to html equivalents')
58-
flog.debug('Leave')
59-
return text
60-
61-
6224
def documented_type(text) -> str:
6325
''' Convert any python type to a type for documentation '''
6426

@@ -80,89 +42,6 @@ def do_max(seq):
8042
return max(seq)
8143

8244

83-
# In the following, we make heavy use of escaped whitespace ("\ ") being removed from the output.
84-
# See
85-
# https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html#character-level-inline-markup-1
86-
# for further information.
87-
88-
def _rst_ify_italic(m: 're.Match') -> str:
89-
return f"\\ :emphasis:`{rst_escape(m.group(1), escape_ending_whitespace=True)}`\\ "
90-
91-
92-
def _rst_ify_bold(m: 're.Match') -> str:
93-
return f"\\ :strong:`{rst_escape(m.group(1), escape_ending_whitespace=True)}`\\ "
94-
95-
96-
def _rst_ify_module(m: 're.Match') -> str:
97-
fqcn = f'{m.group(1)}.{m.group(2)}.{m.group(3)}'
98-
return f"\\ :ref:`{rst_escape(fqcn)} <ansible_collections.{fqcn}_module>`\\ "
99-
100-
101-
def _escape_url(url: str) -> str:
102-
# We include '<>[]{}' in safe to allow urls such as 'https://<HOST>:[PORT]/v{version}/' to
103-
# remain unmangled by percent encoding
104-
return quote(url, safe=':/#?%<>[]{}')
105-
106-
107-
def _rst_ify_link(m: 're.Match') -> str:
108-
return f"\\ `{rst_escape(m.group(1))} <{_escape_url(m.group(2))}>`__\\ "
109-
110-
111-
def _rst_ify_url(m: 're.Match') -> str:
112-
return f"\\ {_escape_url(m.group(1))}\\ "
113-
114-
115-
def _rst_ify_ref(m: 're.Match') -> str:
116-
return f"\\ :ref:`{rst_escape(m.group(1))} <{m.group(2)}>`\\ "
117-
118-
119-
def _rst_ify_const(m: 're.Match') -> str:
120-
# Escaping does not work in double backticks, so we use the :literal: role instead
121-
return f"\\ :literal:`{rst_escape(m.group(1), escape_ending_whitespace=True)}`\\ "
122-
123-
124-
def rst_ify(text):
125-
''' convert symbols like I(this is in italics) to valid restructured text '''
126-
127-
flog = mlog.fields(func='rst_ify')
128-
flog.fields(text=text).debug('Enter')
129-
_counts = {}
130-
131-
text, _counts['italic'] = _ITALIC.subn(_rst_ify_italic, text)
132-
text, _counts['bold'] = _BOLD.subn(_rst_ify_bold, text)
133-
text, _counts['module'] = _MODULE.subn(_rst_ify_module, text)
134-
text, _counts['link'] = _LINK.subn(_rst_ify_link, text)
135-
text, _counts['url'] = _URL.subn(_rst_ify_url, text)
136-
text, _counts['ref'] = _REF.subn(_rst_ify_ref, text)
137-
text, _counts['const'] = _CONST.subn(_rst_ify_const, text)
138-
text, _counts['ruler'] = _RULER.subn('\n\n.. raw:: html\n\n <hr>\n\n', text)
139-
140-
flog.fields(counts=_counts).info('Number of macros converted to rst equivalents')
141-
flog.debug('Leave')
142-
return text
143-
144-
145-
def rst_escape(value: t.Any, escape_ending_whitespace=False) -> str:
146-
''' make sure value is converted to a string, and RST special characters are escaped '''
147-
148-
if not isinstance(value, str):
149-
value = str(value)
150-
151-
value = value.replace('\\', '\\\\')
152-
value = value.replace('<', '\\<')
153-
value = value.replace('>', '\\>')
154-
value = value.replace('_', '\\_')
155-
value = value.replace('*', '\\*')
156-
value = value.replace('`', '\\`')
157-
158-
if escape_ending_whitespace and value.endswith(' '):
159-
value = value + '\\ '
160-
if escape_ending_whitespace and value.startswith(' '):
161-
value = '\\ ' + value
162-
163-
return value
164-
165-
16645
def rst_fmt(text, fmt):
16746
''' helper for Jinja2 to do format strings '''
16847

0 commit comments

Comments
 (0)