Merge branch 'master' into fix/breaks-extra-typeerror-with-break-on-newline

Author: nicholasserra

CHANGES.md

@@ -9,6 +9,7 @@
 - [pull #701] Allow boolean attribute syntax in `markdown-in-html` extra
 - [pull #704] Fix XSS from smuggling spans into image attributes (#702, #703)
 - [pull #710] Add emoji support (#709)
+- [pull #713] Fix `header-ids` extra generating duplicate ids when a suffixed id collides with another header (#661)
 
 
 ## python-markdown2 2.5.5

lib/markdown2.py

@@ -419,6 +419,8 @@ def _setup_extras(self):
         if "header-ids" in self.extras:
             if not hasattr(self, '_count_from_header_id') or self.extras['header-ids'].get('reset-count', False):
                 self._count_from_header_id = defaultdict(int)
+            if not hasattr(self, '_header_ids_seen') or self.extras['header-ids'].get('reset-count', False):
+                self._header_ids_seen = set()
         if "metadata" in self.extras:
             self.metadata: dict[str, Any] = {}
 
@@ -638,12 +640,21 @@ def preprocess(self, text: str) -> str:
     def _extract_metadata(self, text: str) -> str:
         if text.startswith("---"):
             fence_splits = re.split(self._meta_data_fence_pattern, text, maxsplit=2)
+            if len(fence_splits) < 3:
+                # A leading '---' with no closing fence is a horizontal rule (or
+                # unterminated front matter), not metadata. re.split returns
+                # fewer than three elements in that case, so leave text as-is.
+                return text
             metadata_content = fence_splits[1]
             tail = fence_splits[2]
         else:
             metadata_split = re.split(self._meta_data_newline, text, maxsplit=1)
             metadata_content = metadata_split[0]
-            tail = metadata_split[1]
+            # There is no blank line to split on when the whole document is a
+            # single block (e.g. a tab-indented code block with no trailing
+            # blank line), so re.split returns a single element and there is
+            # no document body after the metadata.
+            tail = metadata_split[1] if len(metadata_split) > 1 else ""
 
         # _meta_data_pattern only has one capturing group, so we can assume
         # the returned type to be list[str]
@@ -1589,9 +1600,17 @@ def header_id_from_text(self,
         if prefix and isinstance(prefix, str):
             header_id = prefix + '-' + header_id
 
-        self._count_from_header_id[header_id] += 1
-        if 0 == len(header_id) or self._count_from_header_id[header_id] > 1:
-            header_id += '-%s' % self._count_from_header_id[header_id]
+        base_id = header_id
+        self._count_from_header_id[base_id] += 1
+        if 0 == len(base_id) or self._count_from_header_id[base_id] > 1:
+            header_id = '%s-%s' % (base_id, self._count_from_header_id[base_id])
+        # A suffixed id may still collide with a differently-named header
+        # (e.g. "# Chapter" twice yields "chapter-2", which clashes with
+        # "# Chapter 2"). Keep bumping until the id is genuinely unique.
+        while header_id in self._header_ids_seen:
+            self._count_from_header_id[base_id] += 1
+            header_id = '%s-%s' % (base_id, self._count_from_header_id[base_id])
+        self._header_ids_seen.add(header_id)
 
         return header_id
 

test/test_markdown2.py

@@ -295,6 +295,41 @@ def test_breaks_and_break_on_newline_together(self):
                         'break-on-newline': None}),
             '<p>a<br />\nb</p>\n')
     test_breaks_and_break_on_newline_together.tags = ["breaks", "extras"]
+    
+    def test_metadata_no_blank_line(self):
+        # A single-block document with no blank line (e.g. a tab-indented code
+        # block) gives the metadata extra nothing to split on, so re.split
+        # returns a single element. This used to raise IndexError; it should
+        # render normally with no metadata extracted.
+        result = markdown2.markdown('\tsome indented code', extras=['metadata'])
+        self.assertEqual(result, '<pre><code>some indented code\n</code></pre>\n')
+        self.assertEqual(result.metadata, {})
+    test_metadata_no_blank_line.tags = ["metadata", "issue"]
+
+    def test_metadata_leading_hr_no_closing_fence(self):
+        # A document that opens with '---' (a horizontal rule) but has no
+        # closing '---' fence is not front matter. This used to raise
+        # IndexError; it should render the '---' as an <hr>.
+        result = markdown2.markdown('---\n# My Document\n', extras=['metadata'])
+        self.assertEqual(result, '<hr />\n\n<h1>My Document</h1>\n')
+        self.assertEqual(result.metadata, {})
+    test_metadata_leading_hr_no_closing_fence.tags = ["metadata", "issue"]
+
+    def test_metadata_fenced_front_matter_still_parsed(self):
+        # A complete '---' fenced front-matter block is still extracted.
+        result = markdown2.markdown('---\ntitle: Hi\n---\n# Body\n',
+                                    extras=['metadata'])
+        self.assertEqual(result.metadata, {'title': 'Hi'})
+        self.assertEqual(result, '<h1>Body</h1>\n')
+    test_metadata_fenced_front_matter_still_parsed.tags = ["metadata"]
+
+    def test_metadata_still_parsed_without_fence(self):
+        # The fix must not change how leading key: value metadata is parsed.
+        result = markdown2.markdown('title: Hello\nauthor: Me\n\n# Body\n',
+                                    extras=['metadata'])
+        self.assertEqual(result.metadata, {'title': 'Hello', 'author': 'Me'})
+        self.assertEqual(result, '<h1>Body</h1>\n')
+    test_metadata_still_parsed_without_fence.tags = ["metadata"]
 
     def test_toc_with_persistent_object(self):
         """

test/tm-cases/header_ids_duplicate.html

@@ -0,0 +1,11 @@
+<h1 id="chapter">Chapter</h1>
+
+<p>Test</p>
+
+<h1 id="chapter-2">Chapter</h1>
+
+<p>Test</p>
+
+<h1 id="chapter-2-2">Chapter 2</h1>
+
+<p>Test</p>

test/tm-cases/header_ids_duplicate.opts

@@ -0,0 +1 @@
+{"extras": ["header-ids"]}

test/tm-cases/header_ids_duplicate.tags

@@ -0,0 +1 @@
+extra header-ids issue661

test/tm-cases/header_ids_duplicate.text

@@ -0,0 +1,11 @@
+# Chapter
+
+Test
+
+# Chapter
+
+Test
+
+# Chapter 2
+
+Test