From 927c0e07a53eb45b2f5f770fca89787d6743efe9 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Mon, 7 Sep 2026 13:32:47 -0600 Subject: [PATCH 01/34] Rewrite emphasis handling - This is a complete rewrite of how emphasis handling is done. - Drop use of multiple regex patterns in run in multiple passes and instead evaluate delimiters, nested or otherwise, and build up HTML elements. - Try to consume tokens as much as possible until a full element is constructed (with children if any). - If an outer set of tokens cannot be resolved, but one or more sub tokens can, render the first sub token span and cache the remaining ones for subsequent reentry and render those until the cache is exhausted. - Two tests results were updated to match new behavior. --- markdown/extensions/legacy_em.py | 28 +- markdown/inlinepatterns.py | 531 +++++++++++++++------- tests/misc/underscores.html | 6 - tests/misc/underscores.txt | 11 - tests/test_syntax/inline/test_emphasis.py | 33 +- 5 files changed, 400 insertions(+), 209 deletions(-) delete mode 100644 tests/misc/underscores.html delete mode 100644 tests/misc/underscores.txt diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index 39efe9a73..a78ebb402 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -14,29 +14,7 @@ from __future__ import annotations from . import Extension -from ..inlinepatterns import UnderscoreProcessor, EmStrongItem, EM_STRONG2_RE, STRONG_EM2_RE -import re - -# _emphasis_ -EMPHASIS_RE = r'(_)([^_]+)\1' - -# __strong__ -STRONG_RE = r'(_{2})(.+?)\1' - -# __strong_em___ -STRONG_EM_RE = r'(_)\1(?!\1)([^_]+?)\1(?!\1)(.+?)\1{3}' - - -class LegacyUnderscoreProcessor(UnderscoreProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" - - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] +from ..inlinepatterns import DelimiterProcessor class LegacyEmExtension(Extension): @@ -50,8 +28,8 @@ def extendMarkdown(self, md): | [`LegacyUnderscoreProcessor`][markdown.extensions.legacy_em.LegacyUnderscoreProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | """ - # flake8: noqa: E501 48-50 - md.inlinePatterns.register(LegacyUnderscoreProcessor(r'_'), 'em_strong2', 50) + # flake8: noqa: E501 27-29 + md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em'), 'em_strong2', 50) def makeExtension(**kwargs): # pragma: no cover diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index d3c3bef26..7ce4efe7c 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -41,7 +41,8 @@ from __future__ import annotations from . import util -from typing import TYPE_CHECKING, Any, Collection, NamedTuple +from typing import TYPE_CHECKING, Any, Collection, NamedTuple, cast +from collections import deque import re import xml.etree.ElementTree as etree from html import entities @@ -89,9 +90,8 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100) inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) - inlinePatterns.register(SimpleTextInlineProcessor(NOT_STRONG_RE), 'not_strong', 70) - inlinePatterns.register(AsteriskProcessor(r'\*'), 'em_strong', 60) - inlinePatterns.register(UnderscoreProcessor(r'_'), 'em_strong2', 50) + inlinePatterns.register(DelimiterProcessor('*', 'strong,em'), 'em_strong', 60) + inlinePatterns.register(DelimiterProcessor('_', 'strong,em', smart=True), 'em_strong2', 50) return inlinePatterns @@ -107,36 +107,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro ESCAPE_RE = r'\\(.)' """ Match a backslash escaped character (`\\<` or `\\*`). """ -EMPHASIS_RE = r'(\*)([^\*]+)\1' -""" Match emphasis with an asterisk (`*emphasis*`). """ - -STRONG_RE = r'(\*{2})(.+?)\1' -""" Match strong with an asterisk (`**strong**`). """ - -SMART_STRONG_RE = r'(?)` or `[text](url "title")`). """ @@ -149,9 +119,6 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro IMAGE_REFERENCE_RE = IMAGE_LINK_RE """ Match start of image reference (`![alt text][2]`). """ -NOT_STRONG_RE = r'((^|(?<=\s))(\*{1,3}|_{1,3})(?=\s|$))' -""" Match a stand-alone `*` or `_`. """ - AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^<>]*)>' """ Match an automatic link (``). """ @@ -596,151 +563,383 @@ def _unescape(m: re.Match[str]) -> str: return RE.sub(_unescape, text) -class AsteriskProcessor(InlineProcessor): - """Emphasis processor for handling strong and em matches inside asterisks.""" - - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(STRONG_EM3_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ +class DelimiterProcessor(InlineProcessor): + """Processor for handling complex nested patterns such as strong and em matches.""" - def build_single(self, m: re.Match[str], tag: str, idx: int) -> etree.Element: - """Return single tag.""" - el1 = etree.Element(tag) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - return el1 - - def build_double(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tag.""" - - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el2, None, idx) - el1.append(el2) - if len(m.groups()) == 3: - text = m.group(3) - self.parse_sub_patterns(text, el1, el2, idx) - return el1 - - def build_double2(self, m: re.Match[str], tags: str, idx: int) -> etree.Element: - """Return double tags (variant 2): `text text`.""" - - tag1, tag2 = tags.split(",") - el1 = etree.Element(tag1) - el2 = etree.Element(tag2) - text = m.group(2) - self.parse_sub_patterns(text, el1, None, idx) - text = m.group(3) - el1.append(el2) - self.parse_sub_patterns(text, el2, None, idx) - return el1 - - def parse_sub_patterns( - self, data: str, parent: etree.Element, last: etree.Element | None, idx: int + def __init__( + self, + token: str, + tags: str, + md: Markdown | None = None, + smart: bool = False, + double: bool = False ) -> None: """ - Parses sub patterns. - - `data`: text to evaluate. + Initialize. - `parent`: Parent to attach text and sub elements to. - - `last`: Last appended child to parent. Can also be None if parent has no children. + Arguments: + token: A single character token. + tags: A tag or two tags seprated by comma. When two are specified, the first will be the + one that takes double tokens. + md: the Markdown object + smart: Enable intelligent word logic. + double: If only one tag is specified, indicate whether it requires double tokens. - `idx`: Current pattern index that was used to evaluate the parent. """ - offset = 0 - pos = 0 - - length = len(data) - while pos < length: - # Find the start of potential emphasis or strong tokens - if self.compiled_re.match(data, pos): - matched = False - # See if the we can match an emphasis/strong pattern - for index, item in enumerate(self.PATTERNS): - # Only evaluate patterns that are after what was used on the parent - if index <= idx: - continue - m = item.pattern.match(data, pos) - if m: - # Append child nodes to parent - # Text nodes should be appended to the last - # child if present, and if not, it should - # be added as the parent's text node. - text = data[offset:m.start(0)] - if text: - if last is not None: - last.tail = text - else: - parent.text = text - el = self.build_element(m, item.builder, item.tags, index) - parent.append(el) - last = el - # Move our position past the matched hunk - offset = pos = m.end(0) - matched = True - if not matched: - # We matched nothing, move on to the next character - pos += 1 + # Cache info + self.regions: list[tuple[int, int, int, int, int]] = [] + self.stack: deque[tuple[int, int, int]] = deque() + self.cache_index = 0 + self.cache_pos = 0 + + self.smart = smart + self.tags = tags.split(',') + self.double = len(tags) != 2 and double + super().__init__(self._build_patterns(token), md) + + def _build_patterns(self, token: str) -> str: + """Build regular expression patterns.""" + + # Build up patterns + self.token = token + etoken = re.escape(token) + avoid_start = fr'(?:(?<=_)|(?(?(?{avoid_start}{etoken}{{1,3}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + elif self.double: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{avoid_start}{etoken}{{2}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) else: - # Increment position as no potential emphasis start was found. - pos += 1 - - # Append any leftover text as a text node. - text = data[offset:] - if text: - if last is not None: - last.tail = text + # This case is not currently used + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{avoid_start}{etoken}{{1}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + # Patterns for "dumb" cases. + else: + if len(self.tags) == 2: + self.boundary = re.compile( + fr'''(?x)(?: + (?P(?(?{etoken}{{1,3}}(?![\s{etoken}])(?!$)) + )''', + flags=re.UNICODE + ) + elif self.double: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{etoken}{{2}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) else: - parent.text = text - - def build_element(self, m: re.Match[str], builder: str, tags: str, index: int) -> etree.Element: + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{etoken}{{1}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) + + return fr'{etoken}' + + def _build_element( + self, + data: str, + start: int = 0, + offset: int = 0 + ) -> tuple[etree.Element, int]: """Element builder.""" - if builder == 'double2': - return self.build_double2(m, tags, index) - elif builder == 'double': - return self.build_double(m, tags, index) + regions = self.regions + el: etree.Element | None = None + last: Any = None + previous: Any = None + greater: Any = None + lesser: Any = None + + triple = set() + outer: list[etree.Element] = [] + outer_r: list[tuple[int, int, int, int, int]] = [] + + if len(self.tags) == 2: + greater, lesser = self.tags + elif self.double: + greater = self.tags[0] + lesser = None else: - return self.build_single(m, tags, index) - - def handleMatch(self, m: re.Match[str], data: str) -> tuple[etree.Element | None, int | None, int | None]: - """Parse patterns.""" - - el = None - start = None - end = None - - for index, item in enumerate(self.PATTERNS): - m1 = item.pattern.match(data, m.start(0)) - if m1: - start = m1.start(0) - end = m1.end(0) - el = self.build_element(m1, item.builder, item.tags, index) + lesser = self.tags[0] + greater = None + + # Iterate regions creating the elements they represent + end = len(regions) + idx = 0 + for idx, i in enumerate(range(start, end), 1): + r = regions[i] + # Not contained within region + if idx and r[0] > regions[start][3]: + idx -= 1 break - return el, start, end + # Get the appropriate element(s) + if r[4] == 3: + el1 = etree.Element(greater) + el2 = etree.Element(lesser) + elif r[4] == 2: + el1 = etree.Element(greater) + el2 = None + else: + el1 = etree.Element(lesser) + el2 = None + + # Populate the elements with their text + if idx > 1: + if last.text is None: + if previous[2] < r[0]: + last.text = data[previous[1]+offset:previous[2]+offset] + else: + last.text = data[previous[1]+offset:r[0]+offset] + if last is not outer[-1] and last.tail is None: + if r[0] < outer_r[-1][3]: + last.tail = data[previous[3]+offset:r[0]+offset] + else: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + outer[-1].tail = data[outer_r[-1][3]+offset:r[0]+offset] + + # First element + if el is None: + el = el1 + last = el + outer.append(el) + outer_r.append(r) + + # Subsequent elements + else: + # Is the current outer element no longer wrapping this one? + while len(outer_r) > 1 and r[3] > outer_r[-1][3]: + outer.pop() + outer_r.pop() + # Double nested element (triple token) + if outer[-1] in triple: + outer[-1][-1].append(el1) + + # Non-nested + else: + outer[-1].append(el1) + + # Is this element wrapping the next? + if i + 1 < end: + if r[3] > regions[i + 1][3]: + outer.append(el1) + outer_r.append(r) + + # Track the last element we parsed. + last = el1 + + # Nest secondary element if there is one. + # Track triple tokens (double elements) + # so we can identify quickly and properly nest. + if el2 is not None: + el1.append(el2) + last = el2 + triple.add(el1) + + # Track the previous region. + previous = r + + # Populate remaining elements with their text + while outer: + if last.text is None: + last.text = data[previous[1]+offset:previous[2]+offset] + if last.tail is None and last is not outer[-1]: + last.tail = data[previous[3]+offset:outer_r[-1][2]+offset] + last = outer.pop() + previous = outer_r.pop() + + return cast('etree.Element', el), idx + + def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]: + """Get a cached result.""" + + stack = self.stack + regions = self.regions + + # Process the next region(s) in the cache + offset = pos - self.cache_pos + start, end = regions[self.cache_index][0], regions[self.cache_index][3] + el, count = self._build_element(data, self.cache_index, offset) + + # Determine next offset + self.cache_index += count + if self.cache_index < len(regions): + self.cache_pos = regions[self.cache_index][0] + while stack: + entry = stack.popleft() + if entry[0] > end: + if entry[0] < self.cache_pos: + self.cache_pos = entry[0] + break + + # Nothing left to process + else: + regions.clear() + stack.clear() + self.cache_index = 0 + self.cache_pos = 0 + + # Whether element is valid or not, we'll advance past the end + return el, start + offset, end + offset + + def handleMatch( # type: ignore[override] + self, + m: re.Match[str], + data: str + ) -> tuple[etree.Element | None, int | None, int | None]: + """Parse delimiter pattern.""" + + # Do we have entries we haven't returned yet? + if self.regions: + return self.get_cached_result(m.start(0), data) + + # If token is not an opening, quit + m2 = self.boundary.match(data, m.start(0)) + if m2 is None or m2.lastgroup[0] == 'e': # type: ignore[index] + if m2 is not None: + m = m2 + # Advance past the full length of the delimiter found + return None, m.start(0), m.end(0) + + # Get the stack and regions + stack = self.stack + regions = self.regions + + # Data offset + offset = m2.end(0) + # Stack of opening delimiters + stack.append((m2.start(0), offset, len(m2.group(0)))) + + # Pair tokens until the stack is empty or we can no longer find tokens. + while stack: + m2 = self.boundary.search(data, offset) + if m2 is None: + break + offset = m2.end(0) + + # Get current and last delimiter size + current = len(m2.group(0)) + last = stack[-1][-1] + + # Some delimiters may be ambiguous and look like both a start or an end + is_start = m2.lastgroup[0] != 'e' # type: ignore[index] + is_end = not is_start or m2.lastgroup[0] != 's' # type: ignore[index] + ambiguous = is_start and is_end + + # Find closing tokens + # Looking for: + # - `*em*` + # - `**strong**` + # - `***strong,em***` + # - `*em**` + # - `*em***` + # - `**strong***` + # + # Avoid ambiguous tokens that could be a start or an end. + # Consume starts until the end token is fully consumed. + # If we don't consume the entire end, see if next rule consumes it. + if is_end and ((not ambiguous and current > last) or (current == last)): + is_start = False + + # Consume previous tokens until the delimiter is consumed + s = m2.start(0) + while current and last <= current: + delimiter = stack.pop() + + # Build up region for pair and adjust accounting. + regions.append((delimiter[0], delimiter[1], s, s + delimiter[-1], delimiter[-1])) + s += delimiter[-1] + current -= delimiter[-1] + if not stack: + is_end = False + break + last = stack[-1][-1] + + # Do we still have more to consume? + is_end = current and stack and last > current + + # Looking for: + # - `***em*` + # - `***strong**` + # - `**em*` + if is_end and (last == 3 or not ambiguous) and last > current: + is_start = False + delimiter = stack.pop() + new = last - current + regions.append((delimiter[0] + new, delimiter[1], m2.start(0), offset, current)) + stack.append((delimiter[0], delimiter[0] + new, new)) + + # Find opening tokens + if is_start: + # Looking for: + # - `*em ...*` + # - `**strong ...*` + # - `***em ...*` + stack.append((m2.start(0), m2.end(0), current)) + + # Build the HTML elements + if regions: + # Regions may be out of order. + regions.sort(key=lambda x: x[0]) + start, end = regions[0][0], regions[0][3] + el, count = self._build_element(data) + + # Cache unprocessed regions to avoid repeated searches + if count < len(regions): + self.cache_index = count + self.cache_pos = self.regions[count][0] + while stack: + entry = stack.popleft() + if entry[0] > end: + if entry[0] < self.cache_pos: + self.cache_pos = entry[0] + break + else: + # Cleanup + stack.clear() + regions.clear() -class UnderscoreProcessor(AsteriskProcessor): - """Emphasis processor for handling strong and em matches inside underscores.""" + return el, start, end - PATTERNS = [ - EmStrongItem(re.compile(EM_STRONG2_RE, re.DOTALL | re.UNICODE), 'double', 'strong,em'), - EmStrongItem(re.compile(STRONG_EM2_RE, re.DOTALL | re.UNICODE), 'double', 'em,strong'), - EmStrongItem(re.compile(SMART_STRONG_EM_RE, re.DOTALL | re.UNICODE), 'double2', 'strong,em'), - EmStrongItem(re.compile(SMART_STRONG_RE, re.DOTALL | re.UNICODE), 'single', 'strong'), - EmStrongItem(re.compile(SMART_EMPHASIS_RE, re.DOTALL | re.UNICODE), 'single', 'em') - ] - """ The various strong and emphasis patterns handled by this processor. """ + # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. + start = m.start(0) + end = stack[-1][1] if stack else m.end(0) + stack.clear() + return None, start, end class LinkInlineProcessor(InlineProcessor): diff --git a/tests/misc/underscores.html b/tests/misc/underscores.html deleted file mode 100644 index 72d51b8b5..000000000 --- a/tests/misc/underscores.html +++ /dev/null @@ -1,6 +0,0 @@ -

THIS_SHOULD_STAY_AS_IS

-

Here is some emphasis, ok?

-

Ok, at least this should work.

-

THIS__SHOULD__STAY

-

Here is some strong stuff.

-

THISSHOULDSTAY?

\ No newline at end of file diff --git a/tests/misc/underscores.txt b/tests/misc/underscores.txt deleted file mode 100644 index 3c7f4bdd9..000000000 --- a/tests/misc/underscores.txt +++ /dev/null @@ -1,11 +0,0 @@ -THIS_SHOULD_STAY_AS_IS - -Here is some _emphasis_, ok? - -Ok, at least _this_ should work. - -THIS__SHOULD__STAY - -Here is some __strong__ stuff. - -THIS___SHOULD___STAY? diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 6e96ea32c..a7b3c56fa 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -20,6 +20,7 @@ """ from markdown.test_tools import TestCase +import textwrap class TestNotEmphasis(TestCase): @@ -147,7 +148,7 @@ def test_complex_emphasis_smart_underscore(self): def test_complex_emphasis_smart_underscore_mid_word(self): self.assertMarkdownRenders( 'This is text __bold_italic bold___ with more text', - '

This is text __bold_italic bold___ with more text

' + '

This is text bold_italic bold_ with more text

' ) def test_nested_emphasis(self): @@ -191,3 +192,33 @@ def test_link_emphasis_inner_outer(self): '**[**text**](url)**', '

text

' ) + + def test_underscore_legacy(self): + + self.assertMarkdownRenders( + textwrap.dedent( + """ + THIS_SHOULD_STAY_AS_IS + + Here is some _emphasis_, ok? + + Ok, at least _this_ should work. + + THIS__SHOULD__STAY + + Here is some __strong__ stuff. + + THIS___SHOULD___STAY? + """ + ), + textwrap.dedent( + """ +

THIS_SHOULD_STAY_AS_IS

+

Here is some emphasis, ok?

+

Ok, at least this should work.

+

THIS__SHOULD__STAY

+

Here is some strong stuff.

+

THIS___SHOULD___STAY?

+ """ + ).strip() + ) From c06c76bffb005003333725e43b8a1b3f303c64c0 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 06:44:37 -0600 Subject: [PATCH 02/34] Remove outdated comment --- markdown/inlinepatterns.py | 1 - 1 file changed, 1 deletion(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 7ce4efe7c..12211dc9c 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -813,7 +813,6 @@ def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, in self.cache_index = 0 self.cache_pos = 0 - # Whether element is valid or not, we'll advance past the end return el, start + offset, end + offset def handleMatch( # type: ignore[override] From 190dda30c1b4b6a96e8ec17f9afde46d87eedc4c Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 06:59:59 -0600 Subject: [PATCH 03/34] If Markdown object is ever left in a bad state reset the stack This is a prevented measure to ensure the processor is always in a good state. This situation has never been observed, but if it did occur, this would allow the processor to reset its state and continue properly. Only extensions have a way to reset, processors don't. Nor do they have a way to detect when a reset would be needed. - Add a current time for each Markdown run. - Have DelimiterProcessor compare the each run time, and if it has changed, perform a reset of the stack. --- markdown/core.py | 3 +++ markdown/extensions/legacy_em.py | 2 +- markdown/inlinepatterns.py | 12 ++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/markdown/core.py b/markdown/core.py index 370cb7ec5..fd946d54f 100644 --- a/markdown/core.py +++ b/markdown/core.py @@ -20,6 +20,7 @@ from __future__ import annotations import codecs +import time import sys import logging import importlib @@ -106,6 +107,7 @@ def __init__(self, **kwargs): """ + self.last_run: float = 0.0 self.tab_length: int = kwargs.get('tab_length', 4) self.ESCAPED_CHARS: list[str] = [ @@ -267,6 +269,7 @@ def reset(self) -> Markdown: Called once upon creation of a class instance. Should be called manually between calls to [`Markdown.convert`][markdown.Markdown.convert]. """ + self.last_run = time.time() self.htmlStash.reset() self.references.clear() diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index a78ebb402..3a9b02e1d 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -29,7 +29,7 @@ def extendMarkdown(self, md): """ # flake8: noqa: E501 27-29 - md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em'), 'em_strong2', 50) + md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em', md), 'em_strong2', 50) def makeExtension(**kwargs): # pragma: no cover diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 12211dc9c..9c985c244 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -90,8 +90,8 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100) inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) - inlinePatterns.register(DelimiterProcessor('*', 'strong,em'), 'em_strong', 60) - inlinePatterns.register(DelimiterProcessor('_', 'strong,em', smart=True), 'em_strong2', 50) + inlinePatterns.register(DelimiterProcessor('*', 'strong,em', md), 'em_strong', 60) + inlinePatterns.register(DelimiterProcessor('_', 'strong,em', md, smart=True), 'em_strong2', 50) return inlinePatterns @@ -593,6 +593,7 @@ def __init__( self.cache_index = 0 self.cache_pos = 0 + self.last_run = 0.0 self.smart = smart self.tags = tags.split(',') self.double = len(tags) != 2 and double @@ -822,6 +823,13 @@ def handleMatch( # type: ignore[override] ) -> tuple[etree.Element | None, int | None, int | None]: """Parse delimiter pattern.""" + # We are in a new run. Reset just in case we were somehow left in a bad state. + if self.md.last_run != self.last_run: + self.regions.clear() + self.stack.clear() + self.cache_index = 0 + self.cache_pos = 0 + # Do we have entries we haven't returned yet? if self.regions: return self.get_cached_result(m.start(0), data) From 6fca854a113a48ddedaac70042b5855fd9684f83 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 07:25:18 -0600 Subject: [PATCH 04/34] Consolidate reset behavior and ensure we update our tracked run --- markdown/inlinepatterns.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 9c985c244..9224d588a 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -587,18 +587,22 @@ def __init__( """ - # Cache info - self.regions: list[tuple[int, int, int, int, int]] = [] - self.stack: deque[tuple[int, int, int]] = deque() - self.cache_index = 0 - self.cache_pos = 0 - self.last_run = 0.0 self.smart = smart self.tags = tags.split(',') self.double = len(tags) != 2 and double + self.reset() super().__init__(self._build_patterns(token), md) + def reset(self): + """Rest.""" + + # Cache info + self.regions: list[tuple[int, int, int, int, int]] = [] + self.stack: deque[tuple[int, int, int]] = deque() + self.cache_index = 0 + self.cache_pos = 0 + def _build_patterns(self, token: str) -> str: """Build regular expression patterns.""" @@ -809,10 +813,7 @@ def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, in # Nothing left to process else: - regions.clear() - stack.clear() - self.cache_index = 0 - self.cache_pos = 0 + self.reset() return el, start + offset, end + offset @@ -825,10 +826,8 @@ def handleMatch( # type: ignore[override] # We are in a new run. Reset just in case we were somehow left in a bad state. if self.md.last_run != self.last_run: - self.regions.clear() - self.stack.clear() - self.cache_index = 0 - self.cache_pos = 0 + self.last_run = self.md.last_run + self.reset() # Do we have entries we haven't returned yet? if self.regions: From fb46bb35be3d71a24653bf8c030a406d59f5ca60 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 07:27:37 -0600 Subject: [PATCH 05/34] Have other reset locations call the reset function as well --- markdown/inlinepatterns.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 9224d588a..d4c47f2d4 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -936,15 +936,14 @@ def handleMatch( # type: ignore[override] break else: # Cleanup - stack.clear() - regions.clear() + self.reset() return el, start, end # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. start = m.start(0) end = stack[-1][1] if stack else m.end(0) - stack.clear() + self.reset() return None, start, end From 3475564c816a311f136811086459bad4ad4af7df Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 08:04:49 -0600 Subject: [PATCH 06/34] Add return type --- markdown/inlinepatterns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index d4c47f2d4..52294e713 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -594,7 +594,7 @@ def __init__( self.reset() super().__init__(self._build_patterns(token), md) - def reset(self): + def reset(self) -> None: """Rest.""" # Cache info From 96b9e4bca40ba07554fed4798bcbf539a5aed657 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 08:27:01 -0600 Subject: [PATCH 07/34] Fix link errors --- docs/extensions/api.md | 2 +- markdown/extensions/legacy_em.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/extensions/api.md b/docs/extensions/api.md index 6952bcd19..4822bbc2b 100644 --- a/docs/extensions/api.md +++ b/docs/extensions/api.md @@ -349,7 +349,7 @@ Here are some convenience functions and other examples: | Class | Kind | Description | | -------------------------------------------------------------------------------------|-----------|---------------------------------------------------------------| -| [`AsteriskProcessor`][markdown.inlinepatterns.AsteriskProcessor] | built-in | Emphasis processor for handling strong and em matches inside asterisks | +| [`DelimiterProcessor`][markdown.inlinepatterns.DelimiterProcessor] | built-in | Emphasis processor for handling strong and em matches | | [`WikiLinksInlineProcessor`][markdown.extensions.wikilinks.WikiLinksInlineProcessor] | extension | Link `[[article names]]` to wiki given in metadata | | [`FootnoteInlineProcessor`][markdown.extensions.footnotes.FootnoteInlineProcessor] | extension | Replaces footnote in text with link to footnote div at bottom | diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index 3a9b02e1d..e0edf05b4 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -23,9 +23,9 @@ class LegacyEmExtension(Extension): def extendMarkdown(self, md): """ Register the processor. - | Class Instance | Registry | Name | Priority | - | ------------------------------------------------------------- | ---------------------------------------------------------------- | ------ | :------: | - | [`LegacyUnderscoreProcessor`][markdown.extensions.legacy_em.LegacyUnderscoreProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | + | Class Instance | Registry | Name | Priority | + | ------------------------------------------------------------------ | ---------------------------------------------------------------- | ------------ | :------: | + | [`DelimiterProcessor`][markdown.inlinepatterns.DelimiterProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | """ # flake8: noqa: E501 27-29 From f7bb325979ab4c2bbfcabeddebd738cb497722a6 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 8 Sep 2026 08:30:52 -0600 Subject: [PATCH 08/34] Add a changelog item --- docs/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.md b/docs/changelog.md index a14c77293..618bce09a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -20,6 +20,7 @@ See the [Contributing Guide](contributing.md) for details. same paragraph, instead of splitting the paragraph around it (#1643). * Fix issue where inline HTML attributes were rejected if they had `<` or `>` in the attribute (#1647). * Fix issue where an unterminated end tag (` Date: Wed, 9 Sep 2026 10:03:55 -0600 Subject: [PATCH 09/34] Consolidate duplicate code --- markdown/inlinepatterns.py | 51 ++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 52294e713..0f5b3ab15 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -789,32 +789,37 @@ def _build_element( return cast('etree.Element', el), idx - def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]: - """Get a cached result.""" - - stack = self.stack - regions = self.regions + def increment_next_position(self, end: int, count: int) -> None: + """ + Increment cache position to the next location that we can initiate an insertion. - # Process the next region(s) in the cache - offset = pos - self.cache_pos - start, end = regions[self.cache_index][0], regions[self.cache_index][3] - el, count = self._build_element(data, self.cache_index, offset) + Cache position should be the first match after our current replacement. + This gives us an anchor to calculate the new offset after insertion. + """ # Determine next offset self.cache_index += count - if self.cache_index < len(regions): - self.cache_pos = regions[self.cache_index][0] - while stack: - entry = stack.popleft() + if self.cache_index < len(self.regions): + self.cache_pos = self.regions[self.cache_index][0] + while self.stack: + entry = self.stack.popleft() if entry[0] > end: - if entry[0] < self.cache_pos: - self.cache_pos = entry[0] + self.cache_pos = entry[0] break # Nothing left to process else: self.reset() + def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, int]: + """Get a cached result.""" + + # Process the next region(s) in the cache + regions = self.regions + offset = pos - self.cache_pos + start, end = regions[self.cache_index][0], regions[self.cache_index][3] + el, count = self._build_element(data, self.cache_index, offset) + self.increment_next_position(end, count) return el, start + offset, end + offset def handleMatch( # type: ignore[override] @@ -923,21 +928,7 @@ def handleMatch( # type: ignore[override] regions.sort(key=lambda x: x[0]) start, end = regions[0][0], regions[0][3] el, count = self._build_element(data) - - # Cache unprocessed regions to avoid repeated searches - if count < len(regions): - self.cache_index = count - self.cache_pos = self.regions[count][0] - while stack: - entry = stack.popleft() - if entry[0] > end: - if entry[0] < self.cache_pos: - self.cache_pos = entry[0] - break - else: - # Cleanup - self.reset() - + self.increment_next_position(end, count) return el, start, end # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. From 70785a665db76a39070097f3ac4b265c77e51a0a Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 09:28:19 -0600 Subject: [PATCH 10/34] Fix some edge cases - Fix an issue in element building where we should have exited - When an ambiguous `***` is partially consumed by a smaller start, the remainder should be a non-ambiguous start. - Don't consume an ambiguous opening with an end that is smaller than the opening. - `***` are always consumed, even when ambiguous. --- markdown/inlinepatterns.py | 56 ++++++++++++-------- tests/test_syntax/inline/test_emphasis.py | 62 +++++++++++++++++++++++ 2 files changed, 98 insertions(+), 20 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 0f5b3ab15..b1e1e563d 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -599,7 +599,7 @@ def reset(self) -> None: # Cache info self.regions: list[tuple[int, int, int, int, int]] = [] - self.stack: deque[tuple[int, int, int]] = deque() + self.stack: deque[tuple[int, int, bool, int]] = deque() self.cache_index = 0 self.cache_pos = 0 @@ -623,7 +623,7 @@ def _build_patterns(self, token: str) -> str: ''', flags=re.UNICODE ) - elif self.double: + elif self.double: # pragma: no cover self.boundary = re.compile( fr'''(?x) (?P(? str: ''', flags=re.UNICODE ) - else: + else: # pragma: no cover # This case is not currently used self.boundary = re.compile( fr'''(?x) @@ -653,7 +653,7 @@ def _build_patterns(self, token: str) -> str: )''', flags=re.UNICODE ) - elif self.double: + elif self.double: # pragma: no cover self.boundary = re.compile( fr'''(?x) (?P(? str: ''', flags=re.UNICODE ) - else: + else: # pragma: no cover self.boundary = re.compile( fr'''(?x) (?P(? regions[start][3]: + if idx and r[0] >= regions[start][3]: idx -= 1 break # Get the appropriate element(s) @@ -789,7 +789,7 @@ def _build_element( return cast('etree.Element', el), idx - def increment_next_position(self, end: int, count: int) -> None: + def increment_next_position(self, start: int, count: int) -> None: """ Increment cache position to the next location that we can initiate an insertion. @@ -803,7 +803,7 @@ def increment_next_position(self, end: int, count: int) -> None: self.cache_pos = self.regions[self.cache_index][0] while self.stack: entry = self.stack.popleft() - if entry[0] > end: + if entry[0] > start: self.cache_pos = entry[0] break @@ -819,7 +819,7 @@ def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, in offset = pos - self.cache_pos start, end = regions[self.cache_index][0], regions[self.cache_index][3] el, count = self._build_element(data, self.cache_index, offset) - self.increment_next_position(end, count) + self.increment_next_position(start, count) return el, start + offset, end + offset def handleMatch( # type: ignore[override] @@ -853,7 +853,8 @@ def handleMatch( # type: ignore[override] # Data offset offset = m2.end(0) # Stack of opening delimiters - stack.append((m2.start(0), offset, len(m2.group(0)))) + is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] + stack.append((m2.start(0), offset, is_ambiguous, len(m2.group(0)))) # Pair tokens until the stack is empty or we can no longer find tokens. while stack: @@ -869,7 +870,7 @@ def handleMatch( # type: ignore[override] # Some delimiters may be ambiguous and look like both a start or an end is_start = m2.lastgroup[0] != 'e' # type: ignore[index] is_end = not is_start or m2.lastgroup[0] != 's' # type: ignore[index] - ambiguous = is_start and is_end + is_ambiguous = is_start and is_end # Find closing tokens # Looking for: @@ -883,11 +884,12 @@ def handleMatch( # type: ignore[override] # Avoid ambiguous tokens that could be a start or an end. # Consume starts until the end token is fully consumed. # If we don't consume the entire end, see if next rule consumes it. - if is_end and ((not ambiguous and current > last) or (current == last)): + if is_end and ((not is_ambiguous and current > last) or current in (last, 3)): is_start = False # Consume previous tokens until the delimiter is consumed s = m2.start(0) + original = current while current and last <= current: delimiter = stack.pop() @@ -900,19 +902,33 @@ def handleMatch( # type: ignore[override] break last = stack[-1][-1] + # Should remainder be treated as a new start? + if original == 3 and current and is_ambiguous: + self.stack.append((m2.start(0) + regions[-1][-1], m2.end(0), False, current)) + is_end = False + # Do we still have more to consume? - is_end = current and stack and last > current + else: + is_end = current and stack and last > current # Looking for: # - `***em*` # - `***strong**` # - `**em*` - if is_end and (last == 3 or not ambiguous) and last > current: - is_start = False + if is_end and (last == 3 or not is_ambiguous) and last > current: delimiter = stack.pop() + + # Don't pair with an ambiguous opening + while stack and delimiter[-1] != 3 and delimiter[2]: + delimiter = stack.pop() + last = delimiter[-1] + if delimiter[2]: + break + + is_start = False new = last - current regions.append((delimiter[0] + new, delimiter[1], m2.start(0), offset, current)) - stack.append((delimiter[0], delimiter[0] + new, new)) + stack.append((delimiter[0], delimiter[0] + new, False, new)) # Find opening tokens if is_start: @@ -920,7 +936,7 @@ def handleMatch( # type: ignore[override] # - `*em ...*` # - `**strong ...*` # - `***em ...*` - stack.append((m2.start(0), m2.end(0), current)) + stack.append((m2.start(0), m2.end(0), is_ambiguous, current)) # Build the HTML elements if regions: @@ -928,7 +944,7 @@ def handleMatch( # type: ignore[override] regions.sort(key=lambda x: x[0]) start, end = regions[0][0], regions[0][3] el, count = self._build_element(data) - self.increment_next_position(end, count) + self.increment_next_position(start, count) return el, start, end # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index a7b3c56fa..9909b7790 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -222,3 +222,65 @@ def test_underscore_legacy(self): """ ).strip() ) + + def test_advanced_nesting(self): + + self.maxDiff = None + + self.assertMarkdownRenders( + textwrap.dedent( + """ + **a*bc** + + *a**b**c**d**e**f* + + ***a**b*cd**e*f*** + + ***a**b*cd*e**f*** + + ***a**b*cd*e**f*g*h*** + + ***a***bc**d*e*** + + *a**b**c**d**e**f* + + *a**b***c**d***e**f* + + *a**b***c**d***e**f** + + __a _b c__ + + _a __b __c __d __e __f_ + + ___a __b _c d__ e_ f___ + + ___a __b _c d_ e__ f___ + + ___a __b _c d_ e__ f _g_ h___ + + ___a ___b c__ d_ e___ + + _a __b__ _c __d__ _e __f__ + """ + ), + textwrap.dedent( + """ +

*abc

+

abcde**f

+

abcdef

+

abcdef

+

abcdefgh

+

abcde

+

abcde**f

+

abcde**f

+

abcd*ef

+

_a b c

+

_a __b __c __d __e _f

+

a b c d e f

+

a b c d e f

+

a b c d e f g h

+

a b c d e

+

_a b _c d _e f

+ """ + ).strip() + ) From e8dec3af6c39b5345e66a6840ade8f431018318e Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 10:33:29 -0600 Subject: [PATCH 11/34] Separate cache info defines from reset --- markdown/inlinepatterns.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index b1e1e563d..f14c48aeb 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -587,6 +587,13 @@ def __init__( """ + # Cache info + self.regions: list[tuple[int, int, int, int, int]] = [] + self.stack: deque[tuple[int, int, bool, int]] = deque() + self.cache_index = 0 + self.cache_pos = 0 + self.cache_legacy_pos = -1 + self.last_run = 0.0 self.smart = smart self.tags = tags.split(',') @@ -597,9 +604,8 @@ def __init__( def reset(self) -> None: """Rest.""" - # Cache info - self.regions: list[tuple[int, int, int, int, int]] = [] - self.stack: deque[tuple[int, int, bool, int]] = deque() + self.regions.clear() + self.stack.clear() self.cache_index = 0 self.cache_pos = 0 From b9ffc0d262b1ba789d474b4ff1c58f7c54f6886b Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 10:34:52 -0600 Subject: [PATCH 12/34] Fix lint --- markdown/inlinepatterns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index f14c48aeb..14d2f5540 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -859,7 +859,7 @@ def handleMatch( # type: ignore[override] # Data offset offset = m2.end(0) # Stack of opening delimiters - is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] + is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] stack.append((m2.start(0), offset, is_ambiguous, len(m2.group(0)))) # Pair tokens until the stack is empty or we can no longer find tokens. @@ -926,7 +926,7 @@ def handleMatch( # type: ignore[override] # Don't pair with an ambiguous opening while stack and delimiter[-1] != 3 and delimiter[2]: - delimiter = stack.pop() + delimiter = stack.pop() last = delimiter[-1] if delimiter[2]: break From 935733f132fb429b6a10410c10b390ef09ced1fd Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 11:00:46 -0600 Subject: [PATCH 13/34] Fix a possible scenario where the next region is contiguous --- markdown/inlinepatterns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 14d2f5540..37e78d3bc 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -822,7 +822,7 @@ def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, in # Process the next region(s) in the cache regions = self.regions - offset = pos - self.cache_pos + offset = pos - self.cache_pos if pos != self.cache_pos else pos - regions[self.cache_index][0] start, end = regions[self.cache_index][0], regions[self.cache_index][3] el, count = self._build_element(data, self.cache_index, offset) self.increment_next_position(start, count) From f5145af9acc586d6c569994dc191c9fac57c086b Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 14:03:35 -0600 Subject: [PATCH 14/34] Rework how resetting is performed --- markdown/core.py | 10 +++++++-- markdown/inlinepatterns.py | 10 ++------- tests/test_syntax/inline/test_emphasis.py | 27 +++++++++++++++++++++++ 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/markdown/core.py b/markdown/core.py index fd946d54f..eab4e57b0 100644 --- a/markdown/core.py +++ b/markdown/core.py @@ -29,7 +29,7 @@ from .preprocessors import build_preprocessors from .blockprocessors import build_block_parser from .treeprocessors import build_treeprocessors -from .inlinepatterns import build_inlinepatterns +from .inlinepatterns import build_inlinepatterns, DelimiterProcessor from .postprocessors import build_postprocessors from .extensions import Extension from .serializers import to_html_string, to_xhtml_string @@ -120,6 +120,7 @@ def __init__(self, **kwargs): self.registeredExtensions: list[Extension] = [] self.docType = "" # TODO: Maybe delete this. It does not appear to be used anymore. self.stripTopLevelTags: bool = True + self.delimiters: dict[str, DelimiterProcessor] = {} self.build_parser() @@ -269,10 +270,15 @@ def reset(self) -> Markdown: Called once upon creation of a class instance. Should be called manually between calls to [`Markdown.convert`][markdown.Markdown.convert]. """ - self.last_run = time.time() self.htmlStash.reset() self.references.clear() + for key in list(self.delimiters): + ext = self.delimiters[key] + ext.reset() + if ext not in self.inlinePatterns: + del self.delimiters[key] + for extension in self.registeredExtensions: if hasattr(extension, 'reset'): extension.reset() diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 37e78d3bc..f123b4463 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -570,7 +570,7 @@ def __init__( self, token: str, tags: str, - md: Markdown | None = None, + md: Markdown, smart: bool = False, double: bool = False ) -> None: @@ -588,17 +588,16 @@ def __init__( """ # Cache info + md.delimiters[token] = self self.regions: list[tuple[int, int, int, int, int]] = [] self.stack: deque[tuple[int, int, bool, int]] = deque() self.cache_index = 0 self.cache_pos = 0 - self.cache_legacy_pos = -1 self.last_run = 0.0 self.smart = smart self.tags = tags.split(',') self.double = len(tags) != 2 and double - self.reset() super().__init__(self._build_patterns(token), md) def reset(self) -> None: @@ -835,11 +834,6 @@ def handleMatch( # type: ignore[override] ) -> tuple[etree.Element | None, int | None, int | None]: """Parse delimiter pattern.""" - # We are in a new run. Reset just in case we were somehow left in a bad state. - if self.md.last_run != self.last_run: - self.last_run = self.md.last_run - self.reset() - # Do we have entries we haven't returned yet? if self.regions: return self.get_cached_result(m.start(0), data) diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 9909b7790..8b3f4d086 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -284,3 +284,30 @@ def test_advanced_nesting(self): """ ).strip() ) + + +class TestProcessorRemoval(TestCase): + + def test_remove_processor(self): + + import markdown + from markdown.inlinepatterns import DelimiterProcessor + + # Remove all delimeter processors + md = markdown.Markdown() + + extensions = md.delimiters.values() + self.assertEqual(len(extensions), 2) + + for ext in md.delimiters.values(): + self.assertTrue(isinstance(ext, DelimiterProcessor)) + + md.inlinePatterns.deregister('em_strong') + md.inlinePatterns.deregister('em_strong2') + + # Call reset which will cause them to remove themselves from being registered + md.reset() + + extensions = md.delimiters.values() + self.assertEqual(len(extensions), 0) + From f8d87d26cd75d710769adb139669441c4714fca8 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 14:05:19 -0600 Subject: [PATCH 15/34] Remove unused module --- markdown/core.py | 1 - tests/test_syntax/inline/test_emphasis.py | 1 - 2 files changed, 2 deletions(-) diff --git a/markdown/core.py b/markdown/core.py index eab4e57b0..a04510c03 100644 --- a/markdown/core.py +++ b/markdown/core.py @@ -20,7 +20,6 @@ from __future__ import annotations import codecs -import time import sys import logging import importlib diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 8b3f4d086..1d8897a2b 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -310,4 +310,3 @@ def test_remove_processor(self): extensions = md.delimiters.values() self.assertEqual(len(extensions), 0) - From 53aca45e66395e3621e268aa84929a6b4e7563bd Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 14:07:51 -0600 Subject: [PATCH 16/34] Spelling --- tests/test_syntax/inline/test_emphasis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 1d8897a2b..750c93d77 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -293,7 +293,7 @@ def test_remove_processor(self): import markdown from markdown.inlinepatterns import DelimiterProcessor - # Remove all delimeter processors + # Remove all delimiter processors md = markdown.Markdown() extensions = md.delimiters.values() From 701a666c80fd7f3c9800b74b41c883bd3fc490c9 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Thu, 10 Sep 2026 15:57:09 -0600 Subject: [PATCH 17/34] Use test harness dedent --- tests/test_syntax/inline/test_emphasis.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 750c93d77..eae15889e 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -20,7 +20,6 @@ """ from markdown.test_tools import TestCase -import textwrap class TestNotEmphasis(TestCase): @@ -196,7 +195,7 @@ def test_link_emphasis_inner_outer(self): def test_underscore_legacy(self): self.assertMarkdownRenders( - textwrap.dedent( + self.dedent( """ THIS_SHOULD_STAY_AS_IS @@ -211,7 +210,7 @@ def test_underscore_legacy(self): THIS___SHOULD___STAY? """ ), - textwrap.dedent( + self.dedent( """

THIS_SHOULD_STAY_AS_IS

Here is some emphasis, ok?

@@ -220,15 +219,13 @@ def test_underscore_legacy(self):

Here is some strong stuff.

THIS___SHOULD___STAY?

""" - ).strip() + ) ) def test_advanced_nesting(self): - self.maxDiff = None - self.assertMarkdownRenders( - textwrap.dedent( + self.dedent( """ **a*bc** @@ -263,7 +260,7 @@ def test_advanced_nesting(self): _a __b__ _c __d__ _e __f__ """ ), - textwrap.dedent( + self.dedent( """

*abc

abcde**f

@@ -282,7 +279,7 @@ def test_advanced_nesting(self):

a b c d e

_a b _c d _e f

""" - ).strip() + ) ) From 366508708d2c74cfca1a44619f281805b00c0819 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Fri, 11 Sep 2026 18:36:39 -0600 Subject: [PATCH 18/34] Handle emphasis token chains longer than 3 - When consuming a chain of tokens greater than 2, consume in increments of 2 until all are consumed or 1 is left, and then consume 1. This means that the previous assumption that triple tokens yield `` is now ``. While the syntax is different the logic is the same. This is a necessary change in order to properly consume tokens. - Small clean up of pattern building - Add common mark tests --- markdown/inlinepatterns.py | 131 ++-- tests/basic/strong-and-em-together.html | 4 - tests/basic/strong-and-em-together.txt | 7 - tests/misc/em_strong_complex.html | 14 - tests/misc/em_strong_complex.txt | 27 - tests/misc/nested-patterns.html | 10 - tests/misc/nested-patterns.txt | 13 - tests/test_syntax/blocks/test_hr.py | 2 +- .../test_syntax/extensions/test_legacy_em.py | 4 +- tests/test_syntax/inline/test_emphasis.py | 573 +++++++++++++++++- 10 files changed, 625 insertions(+), 160 deletions(-) delete mode 100644 tests/basic/strong-and-em-together.html delete mode 100644 tests/basic/strong-and-em-together.txt delete mode 100644 tests/misc/em_strong_complex.html delete mode 100644 tests/misc/em_strong_complex.txt delete mode 100644 tests/misc/nested-patterns.html delete mode 100644 tests/misc/nested-patterns.txt diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index f123b4463..bd35fa7c4 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -616,66 +616,33 @@ def _build_patterns(self, token: str) -> str: etoken = re.escape(token) avoid_start = fr'(?:(?<=_)|(?(?(?{avoid_start}{etoken}{{1,3}}(?![\s{etoken}])(?!$)) - ''', - flags=re.UNICODE - ) - elif self.double: # pragma: no cover - self.boundary = re.compile( - fr'''(?x) - (?P(?(?{avoid_start}{etoken}{{2}}(?![\s{etoken}])(?!$)) - ''', - flags=re.UNICODE - ) - else: # pragma: no cover - # This case is not currently used - self.boundary = re.compile( - fr'''(?x) - (?P(?(?{avoid_start}{etoken}{{1}}(?![\s{etoken}])(?!$)) - ''', - flags=re.UNICODE - ) + self.boundary = re.compile( + fr'''(?x) + (?P(?(?{avoid_start}{etoken}{{{n}}}(?![\s{etoken}])(?!$)) + ''', + flags=re.UNICODE + ) # Patterns for "dumb" cases. else: - if len(self.tags) == 2: - self.boundary = re.compile( - fr'''(?x)(?: - (?P(?(?{etoken}{{1,3}}(?![\s{etoken}])(?!$)) - )''', - flags=re.UNICODE - ) - elif self.double: # pragma: no cover - self.boundary = re.compile( - fr'''(?x) - (?P(?(?{etoken}{{2}}(?![\s{etoken}])(?!$)) - ''', - flags=re.UNICODE - ) - else: # pragma: no cover - self.boundary = re.compile( - fr'''(?x) - (?P(?(?{etoken}{{1}}(?![\s{etoken}])(?!$)) - ''', - flags=re.UNICODE - ) + self.boundary = re.compile( + fr'''(?x)(?: + (?P(?(?{etoken}{{{n}}}(?![\s{etoken}])(?!$)) + )''', + flags=re.UNICODE + ) return fr'{etoken}' @@ -718,8 +685,8 @@ def _build_element( break # Get the appropriate element(s) if r[4] == 3: - el1 = etree.Element(greater) - el2 = etree.Element(lesser) + el1 = etree.Element(lesser) + el2 = etree.Element(greater) elif r[4] == 2: el1 = etree.Element(greater) el2 = None @@ -850,18 +817,19 @@ def handleMatch( # type: ignore[override] stack = self.stack regions = self.regions - # Data offset - offset = m2.end(0) - # Stack of opening delimiters + start = m2.start(0) + end = m2.end(0) + length = end - start is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] - stack.append((m2.start(0), offset, is_ambiguous, len(m2.group(0)))) + self.stack.append((start, start + length, is_ambiguous, length)) # Pair tokens until the stack is empty or we can no longer find tokens. while stack: - m2 = self.boundary.search(data, offset) + m2 = self.boundary.search(data, end) if m2 is None: break - offset = m2.end(0) + start = m2.start(0) + end = m2.end(0) # Get current and last delimiter size current = len(m2.group(0)) @@ -884,27 +852,33 @@ def handleMatch( # type: ignore[override] # Avoid ambiguous tokens that could be a start or an end. # Consume starts until the end token is fully consumed. # If we don't consume the entire end, see if next rule consumes it. - if is_end and ((not is_ambiguous and current > last) or current in (last, 3)): + if ( + is_end and + ((not is_ambiguous and current > last) or (current <= 3 and current == last) or current >= 3) + ): is_start = False # Consume previous tokens until the delimiter is consumed - s = m2.start(0) original = current while current and last <= current: delimiter = stack.pop() # Build up region for pair and adjust accounting. - regions.append((delimiter[0], delimiter[1], s, s + delimiter[-1], delimiter[-1])) - s += delimiter[-1] - current -= delimiter[-1] + size = min(delimiter[-1], 2) + regions.append((delimiter[1] - size, delimiter[1], start, start + size, size)) + start += size + current -= size + if size < delimiter[-1]: + new = delimiter[-1] - size + stack.append((delimiter[0], delimiter[1] - size, delimiter[2], new)) if not stack: is_end = False break last = stack[-1][-1] # Should remainder be treated as a new start? - if original == 3 and current and is_ambiguous: - self.stack.append((m2.start(0) + regions[-1][-1], m2.end(0), False, current)) + if original >= 3 and current and is_ambiguous: + self.stack.append((regions[-1][3], end, False, current)) is_end = False # Do we still have more to consume? @@ -915,7 +889,7 @@ def handleMatch( # type: ignore[override] # - `***em*` # - `***strong**` # - `**em*` - if is_end and (last == 3 or not is_ambiguous) and last > current: + if is_end and (last >= 3 or not is_ambiguous) and last > current: delimiter = stack.pop() # Don't pair with an ambiguous opening @@ -926,9 +900,16 @@ def handleMatch( # type: ignore[override] break is_start = False - new = last - current - regions.append((delimiter[0] + new, delimiter[1], m2.start(0), offset, current)) - stack.append((delimiter[0], delimiter[0] + new, False, new)) + ds, de = delimiter[:2] + while current: + size = min(current, 2) + new = last - size + regions.append((ds + new, de, start, start + size, size)) + start += size + current -= size + last -= size + de -= size + stack.append((ds, de, False, last)) # Find opening tokens if is_start: @@ -936,7 +917,7 @@ def handleMatch( # type: ignore[override] # - `*em ...*` # - `**strong ...*` # - `***em ...*` - stack.append((m2.start(0), m2.end(0), is_ambiguous, current)) + stack.append((start, end, is_ambiguous, current)) # Build the HTML elements if regions: diff --git a/tests/basic/strong-and-em-together.html b/tests/basic/strong-and-em-together.html deleted file mode 100644 index 7bf5163e7..000000000 --- a/tests/basic/strong-and-em-together.html +++ /dev/null @@ -1,4 +0,0 @@ -

This is strong and em.

-

So is this word.

-

This is strong and em.

-

So is this word.

\ No newline at end of file diff --git a/tests/basic/strong-and-em-together.txt b/tests/basic/strong-and-em-together.txt deleted file mode 100644 index 95ee690db..000000000 --- a/tests/basic/strong-and-em-together.txt +++ /dev/null @@ -1,7 +0,0 @@ -***This is strong and em.*** - -So is ***this*** word. - -___This is strong and em.___ - -So is ___this___ word. diff --git a/tests/misc/em_strong_complex.html b/tests/misc/em_strong_complex.html deleted file mode 100644 index 65faddfad..000000000 --- a/tests/misc/em_strong_complex.html +++ /dev/null @@ -1,14 +0,0 @@ -

test test test test

-

test test test test

-

test

-

test

-

test test_

-

test test

-

test_test test_test

-

test test test test

-

test test test test

-

*test

-

test

-

test*

-

test test

-

testtest testtest

\ No newline at end of file diff --git a/tests/misc/em_strong_complex.txt b/tests/misc/em_strong_complex.txt deleted file mode 100644 index 042597184..000000000 --- a/tests/misc/em_strong_complex.txt +++ /dev/null @@ -1,27 +0,0 @@ -___test test__ test test_ - -___test test_ test test__ - -___test___ - -__test__ - -___test_ test___ - -___test_ test__ - -_test_test test_test_ - -***test test** test test* - -***test test* test test** - -**test* - -***test*** - -**test*** - -***test* test** - -*test*test test*test* \ No newline at end of file diff --git a/tests/misc/nested-patterns.html b/tests/misc/nested-patterns.html deleted file mode 100644 index 1c7bb43c6..000000000 --- a/tests/misc/nested-patterns.html +++ /dev/null @@ -1,10 +0,0 @@ -

link -link -link -link -link -link -link

-

I am italic and bold I am just bold

-

Example bold italic on the same line bold italic.

-

Example bold italic on the same line bold italic.

\ No newline at end of file diff --git a/tests/misc/nested-patterns.txt b/tests/misc/nested-patterns.txt deleted file mode 100644 index 9032cf131..000000000 --- a/tests/misc/nested-patterns.txt +++ /dev/null @@ -1,13 +0,0 @@ -___[link](http://example.com)___ -***[link](http://example.com)*** -**[*link*](http://example.com)** -__[_link_](http://example.com)__ -__[*link*](http://example.com)__ -**[_link_](http://example.com)** -[***link***](http://example.com) - -***I am ___italic_ and__ bold* I am `just` bold** - -Example __*bold italic*__ on the same line __*bold italic*__. - -Example **_bold italic_** on the same line **_bold italic_**. diff --git a/tests/test_syntax/blocks/test_hr.py b/tests/test_syntax/blocks/test_hr.py index a8753977b..5aa48465b 100644 --- a/tests/test_syntax/blocks/test_hr.py +++ b/tests/test_syntax/blocks/test_hr.py @@ -318,7 +318,7 @@ def test_hr_after_emstrong(self): ), self.dedent( """ -

text

+

text

""" ) diff --git a/tests/test_syntax/extensions/test_legacy_em.py b/tests/test_syntax/extensions/test_legacy_em.py index ad2862e5e..a1a9f2e5a 100644 --- a/tests/test_syntax/extensions/test_legacy_em.py +++ b/tests/test_syntax/extensions/test_legacy_em.py @@ -55,12 +55,12 @@ def test_complex_multple_underscore_type(self): self.assertMarkdownRenders( 'traced ___along___ bla __blocked__ if other ___or___', - '

traced along bla blocked if other or

' # noqa: E501 + '

traced along bla blocked if other or

' # noqa: E501 ) def test_complex_multple_underscore_type_variant2(self): self.assertMarkdownRenders( 'on the __1-4 row__ of the AP Combat Table ___and___ receive', - '

on the 1-4 row of the AP Combat Table and receive

' + '

on the 1-4 row of the AP Combat Table and receive

' ) diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index eae15889e..127d2c78f 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -161,14 +161,14 @@ def test_complex_multple_emphasis_type(self): self.assertMarkdownRenders( 'traced ***along*** bla **blocked** if other ***or***', - '

traced along bla blocked if other or

' # noqa: E501 + '

traced along bla blocked if other or

' # noqa: E501 ) def test_complex_multple_emphasis_type_variant2(self): self.assertMarkdownRenders( 'on the **1-4 row** of the AP Combat Table ***and*** receive', - '

on the 1-4 row of the AP Combat Table and receive

' + '

on the 1-4 row of the AP Combat Table and receive

' ) def test_link_emphasis_outer(self): @@ -222,6 +222,120 @@ def test_underscore_legacy(self): ) ) + def test_nested_patterns(self): + + self.assertMarkdownRenders( + self.dedent( + """ + ___[link](http://example.com)___ + ***[link](http://example.com)*** + **[*link*](http://example.com)** + __[_link_](http://example.com)__ + __[*link*](http://example.com)__ + **[_link_](http://example.com)** + [***link***](http://example.com) + + ***I am ___italic_ and__ bold* I am `just` bold** + + Example __*bold italic*__ on the same line __*bold italic*__. + + Example **_bold italic_** on the same line **_bold italic_**. + """ + ), + self.dedent( + """ +

link + link + link + link + link + link + link

+

I am italic and bold I am just bold

+

Example bold italic on the same line bold italic.

+

Example bold italic on the same line bold italic.

+ """ # noqa: E501 + ) + ) + + def test_em_strong_complex(self): + + self.assertMarkdownRenders( + self.dedent( + """ + ___test test__ test test_ + + ___test test_ test test__ + + ___test___ + + __test__ + + ___test_ test___ + + ___test_ test__ + + _test_test test_test_ + + ***test test** test test* + + ***test test* test test** + + **test* + + ***test*** + + **test*** + + ***test* test** + + *test*test test*test* + """ + ), + self.dedent( + """ +

test test test test

+

test test test test

+

test

+

test

+

test test_

+

test test

+

test_test test_test

+

test test test test

+

test test test test

+

*test

+

test

+

test*

+

test test

+

testtest testtest

+ """ + ) + ) + + def test_strong_and_em_together(self): + + self.assertMarkdownRenders( + self.dedent( + """ + ***This is strong and em.*** + + So is ***this*** word. + + ___This is strong and em.___ + + So is ___this___ word. + """ + ), + self.dedent( + """ +

This is strong and em.

+

So is this word.

+

This is strong and em.

+

So is this word.

+ """ + ) + ) + def test_advanced_nesting(self): self.assertMarkdownRenders( @@ -267,22 +381,467 @@ def test_advanced_nesting(self):

abcdef

abcdef

abcdefgh

-

abcde

+

abcde

abcde**f

abcde**f

abcd*ef

_a b c

_a __b __c __d __e _f

-

a b c d e f

-

a b c d e f

-

a b c d e f g h

-

a b c d e

+

a b c d e f

+

a b c d e f

+

a b c d e f g h

+

a b c d e

_a b _c d _e f

""" ) ) +class TestCommonMark(TestCase): + """Test CommonMark.""" + + def test_commonmark(self): + """Test CommonMark.""" + + self.maxDiff = None + + self.assertMarkdownRenders( + self.dedent( + R""" + *foo bar* + + a * foo bar* + + + + + + + test * a * + + foo*bar* + + 5*6*78 + + _foo bar_ + + _ foo bar_ + + a_"foo"_ + + foo_bar_ + + 5_6_78 + + пристаням_стремятся_ + + aa_"bb"_cc + + foo-_(bar)_ + + _foo* + + *foo bar * + + *foo bar + * + + + + *(*foo*)* + + *foo*bar + + _foo bar _ + + _(_foo) + + _(_foo_)_ + + _foo_bar + + _пристаням_стремятся + + _foo_bar_baz_ + + _(bar)_. + + **foo bar** + + ** foo bar** + + + + foo**bar** + + __foo bar__ + + __ foo bar__ + + __ + foo bar__ + + a__"foo"__ + + foo__bar__ + + 5__6__78 + + пристаням__стремятся__ + + __foo, __bar__, baz__ + + foo-__(bar)__ + + **foo bar ** + + + + *(**foo**)* + + **Gomphocarpus (*Gomphocarpus physocarpus*, syn. + *Asclepias physocarpa*)** + + **foo "*bar*" foo** + + **foo**bar + + __foo bar __ + + __(__foo) + + _(__foo__)_ + + __foo__bar + + __пристаням__стремятся + + __foo__bar__baz__ + + __(bar)__. + + *foo [bar](/url)* + + *foo + bar* + + _foo __bar__ baz_ + + _foo _bar_ baz_ + + __foo_ bar_ + + *foo *bar** + + *foo **bar** baz* + + *foo**bar**baz* + + *foo**bar* + + ***foo** bar* + + *foo **bar*** + + *foo**bar*** + + foo***bar***baz + + foo******bar*********baz + + *foo **bar *baz* bim** bop* + + *foo [*bar*](/url)* + + ** is not an empty emphasis + + **** is not an empty strong emphasis + + **foo [bar](/url)** + + **foo + bar** + + __foo _bar_ baz__ + + __foo __bar__ baz__ + + ____foo__ bar__ + + **foo **bar**** + + **foo *bar* baz** + + **foo*bar*baz** + + ***foo* bar** + + **foo *bar*** + + **foo *bar **baz** + bim* bop** + + **foo [*bar*](/url)** + + __ is not an empty emphasis + + ____ is not an empty strong emphasis + + foo *** + + foo *\** + + foo *_* + + foo ***** + + foo **\*** + + foo **_** + + **foo* + + *foo** + + ***foo** + + ****foo* + + **foo*** + + *foo**** + + foo ___ + + foo _\__ + + foo _*_ + + foo _____ + + foo __\___ + + foo __*__ + + __foo_ + + _foo__ + + ___foo__ + + ____foo_ + + __foo___ + + _foo____ + + **foo** + + *_foo_* + + __foo__ + + _*foo*_ + + ****foo**** + + ____foo____ + + ******foo****** + + ***foo*** + + _____foo_____ + + *foo _bar* baz_ + + + + **foo **bar baz** + + *foo *bar baz* + + *[bar*](/url) + + _foo [bar_](/url) + + * + + ** + + __ + + *a `*`* + + _a `_`_ + + **a + + __a + """ + ), + self.dedent( + """ +

foo bar

+

a * foo bar*

+ + + + + +

test * a *

+

foobar

+

5678

+

foo bar

+

_ foo bar_

+

a_"foo"_

+

foo_bar_

+

5_6_78

+

пристаням_стремятся_

+

aa_"bb"_cc

+

foo-(bar)

+

_foo*

+

*foo bar *

+

*foo bar + *

+ + +

(foo)

+

foobar

+

_foo bar _

+

_(_foo)

+

(foo)

+

_foo_bar

+

_пристаням_стремятся

+

foo_bar_baz

+

(bar).

+

foo bar

+

** foo bar**

+ + +

foobar

+

foo bar

+

__ foo bar__

+

__ + foo bar__

+

a__"foo"__

+

foo__bar__

+

5__6__78

+

пристаням__стремятся__

+

foo, bar, baz

+

foo-(bar)

+

**foo bar **

+ + +

(foo)

+

Gomphocarpus (Gomphocarpus physocarpus, syn. + Asclepias physocarpa)

+

foo "bar" foo

+

foobar

+

__foo bar __

+

__(__foo)

+

(foo)

+

__foo__bar

+

__пристаням__стремятся

+

foo__bar__baz

+

(bar).

+

foo bar

+

foo + bar

+

foo bar baz

+

foo bar baz

+

foo bar

+

foo bar

+

foo bar baz

+

foobarbaz

+

foo**bar

+

foo bar

+

foo bar

+

foobar

+

foobarbaz

+

foobar***baz

+

foo bar baz bim bop

+

foo bar

+

** is not an empty emphasis

+

**** is not an empty strong emphasis

+

foo bar

+

foo + bar

+

foo bar baz

+

foo bar baz

+

foo bar

+

foo bar

+

foo bar baz

+

foobarbaz

+

foo bar

+

foo bar

+

foo bar baz + bim bop

+

foo bar

+

__ is not an empty emphasis

+

____ is not an empty strong emphasis

+

foo ***

+

foo *

+

foo _

+

foo *****

+

foo *

+

foo _

+

*foo

+

foo*

+

*foo

+

***foo

+

foo*

+

foo***

+

foo ___

+

foo _

+

foo *

+

foo _____

+

foo _

+

foo *

+

_foo

+

foo_

+

_foo

+

___foo

+

foo_

+

foo___

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo

+

foo _bar baz_

+ + +

**foo bar baz

+

*foo bar baz

+

*bar*

+

_foo bar_

+

*

+

**

+

__

+

a *

+

a _

+

**ahttps://foo.bar/?q=**

+

__ahttps://foo.bar/?q=__

+ """ + ) + ) + + class TestProcessorRemoval(TestCase): def test_remove_processor(self): From 2ed5eca5b0d7f4324e1d151ec657dc1ea8b87182 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Fri, 11 Sep 2026 19:45:00 -0600 Subject: [PATCH 19/34] Ignore redundant tests --- tests/test_legacy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_legacy.py b/tests/test_legacy.py index df4da4d71..808def4c6 100644 --- a/tests/test_legacy.py +++ b/tests/test_legacy.py @@ -91,7 +91,7 @@ class TestPl2004(LegacyTestCase): location = os.path.join(parent_test_dir, 'pl/Tests_2004') normalize = True input_ext = '.text' - exclude = ['Yuri_Footnotes', 'Yuri_Attributes'] + exclude = ['Yuri_Footnotes', 'Yuri_Attributes', 'Strong_and_em_together'] class TestPl2007(LegacyTestCase): @@ -116,7 +116,8 @@ class TestPl2007(LegacyTestCase): 'Code_Blocks', 'Links,_reference_style', 'Backslash_escapes', - 'Code_Spans' + 'Code_Spans', + 'Strong_and_em_together' ] From 345402e8f3fe39e7d6b68ba4a888b5d3deb3d6c3 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Sat, 12 Sep 2026 05:49:04 -0600 Subject: [PATCH 20/34] Remove dead code When building elements, we no longer need to account for regions of 3 tokens as they are always broken up into regions less than 3. --- markdown/inlinepatterns.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index bd35fa7c4..f30d6deb4 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -661,7 +661,6 @@ def _build_element( greater: Any = None lesser: Any = None - triple = set() outer: list[etree.Element] = [] outer_r: list[tuple[int, int, int, int, int]] = [] @@ -684,15 +683,10 @@ def _build_element( idx -= 1 break # Get the appropriate element(s) - if r[4] == 3: - el1 = etree.Element(lesser) - el2 = etree.Element(greater) - elif r[4] == 2: + if r[4] == 2: el1 = etree.Element(greater) - el2 = None else: el1 = etree.Element(lesser) - el2 = None # Populate the elements with their text if idx > 1: @@ -722,10 +716,6 @@ def _build_element( outer.pop() outer_r.pop() - # Double nested element (triple token) - if outer[-1] in triple: - outer[-1][-1].append(el1) - # Non-nested else: outer[-1].append(el1) @@ -739,14 +729,6 @@ def _build_element( # Track the last element we parsed. last = el1 - # Nest secondary element if there is one. - # Track triple tokens (double elements) - # so we can identify quickly and properly nest. - if el2 is not None: - el1.append(el2) - last = el2 - triple.add(el1) - # Track the previous region. previous = r From b0eedf5d8968724a00a1b48b03733b6554f46590 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Sat, 12 Sep 2026 09:55:06 -0600 Subject: [PATCH 21/34] Fixes for if just a double or single token is every used We should grab all tokens, regardless of whether of what size we are looking for. Chunk large spans accordingly a delimiter that only requires single delimiters should chunk by one, all others by two. Limit adding new starting spans if a delimiter only requires double tokens. Also fix our double flag logic. --- markdown/inlinepatterns.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index f30d6deb4..87b24e362 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -597,7 +597,7 @@ def __init__( self.last_run = 0.0 self.smart = smart self.tags = tags.split(',') - self.double = len(tags) != 2 and double + self.double = len(self.tags) != 2 and double super().__init__(self._build_patterns(token), md) def reset(self) -> None: @@ -616,20 +616,17 @@ def _build_patterns(self, token: str) -> str: etoken = re.escape(token) avoid_start = fr'(?:(?<=_)|(?(?(?{avoid_start}{etoken}{{{n}}}(?![\s{etoken}])(?!$)) + (?P(?(?{avoid_start}{etoken}{{1,}}(?![\s{etoken}])(?!$)) ''', flags=re.UNICODE ) @@ -637,9 +634,9 @@ def _build_patterns(self, token: str) -> str: else: self.boundary = re.compile( fr'''(?x)(?: - (?P(?(?{etoken}{{{n}}}(?![\s{etoken}])(?!$)) + (?P(?(?{etoken}{{1,}}(?![\s{etoken}])(?!$)) )''', flags=re.UNICODE ) @@ -836,7 +833,7 @@ def handleMatch( # type: ignore[override] # If we don't consume the entire end, see if next rule consumes it. if ( is_end and - ((not is_ambiguous and current > last) or (current <= 3 and current == last) or current >= 3) + ((not is_ambiguous and current > last) or current == last or current >= 3) ): is_start = False @@ -846,11 +843,11 @@ def handleMatch( # type: ignore[override] delimiter = stack.pop() # Build up region for pair and adjust accounting. - size = min(delimiter[-1], 2) + size = min(delimiter[-1], self.max_size) regions.append((delimiter[1] - size, delimiter[1], start, start + size, size)) start += size current -= size - if size < delimiter[-1]: + if size < delimiter[-1] and (not self.double or (delimiter[-1] - size) != 1): new = delimiter[-1] - size stack.append((delimiter[0], delimiter[1] - size, delimiter[2], new)) if not stack: @@ -883,18 +880,19 @@ def handleMatch( # type: ignore[override] is_start = False ds, de = delimiter[:2] - while current: - size = min(current, 2) + while current and (not self.double or current != 1): + size = min(current, self.max_size) new = last - size regions.append((ds + new, de, start, start + size, size)) start += size current -= size last -= size de -= size - stack.append((ds, de, False, last)) + if not self.double or last > 1: + stack.append((ds, de, False, last)) # Find opening tokens - if is_start: + if is_start and (not self.double or current != 1): # Looking for: # - `*em ...*` # - `**strong ...*` From beec4268043532dcba4395c5a594d97e593fc4ef Mon Sep 17 00:00:00 2001 From: facelessuser Date: Sat, 12 Sep 2026 13:38:32 -0600 Subject: [PATCH 22/34] A double width only delimiter processor could fail on single width token A delimiter processor that requires only double length tokens could fail on single width start --- markdown/inlinepatterns.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 87b24e362..c387ca952 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -799,6 +799,11 @@ def handleMatch( # type: ignore[override] start = m2.start(0) end = m2.end(0) length = end - start + + # Double needs at least a size of 2 + if self.double and length < 2: + return None, m.start(0), m.end(0) + is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] self.stack.append((start, start + length, is_ambiguous, length)) From f961e435ab382fab44340872e0519ffef9f53863 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Sat, 12 Sep 2026 14:50:46 -0600 Subject: [PATCH 23/34] Avoid flipping strong/em tag nesting for triple emphasis tokens - This ensures Python Markdown behavior of nested strong/em tags, remain as they were prior to this change, ensuring the the absolute minimal difference between tested cases. --- markdown/inlinepatterns.py | 6 ++- tests/test_legacy.py | 5 +-- tests/test_syntax/blocks/test_hr.py | 2 +- .../test_syntax/extensions/test_legacy_em.py | 4 +- tests/test_syntax/inline/test_emphasis.py | 42 ++++++++++--------- 5 files changed, 32 insertions(+), 27 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index c387ca952..4eb0ec1a7 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -848,7 +848,9 @@ def handleMatch( # type: ignore[override] delimiter = stack.pop() # Build up region for pair and adjust accounting. - size = min(delimiter[-1], self.max_size) + # Special handle span of 3 to preserve old Python Markdown behavior. + # For true CommonMark logic `2` should always be used. + size = min(delimiter[-1], 1 if len(self.tags) == 2 and delimiter[-1] == 3 else self.max_size) regions.append((delimiter[1] - size, delimiter[1], start, start + size, size)) start += size current -= size @@ -886,7 +888,7 @@ def handleMatch( # type: ignore[override] is_start = False ds, de = delimiter[:2] while current and (not self.double or current != 1): - size = min(current, self.max_size) + size = min(current, 1 if len(self.tags) == 2 and current == 3 else self.max_size) new = last - size regions.append((ds + new, de, start, start + size, size)) start += size diff --git a/tests/test_legacy.py b/tests/test_legacy.py index 808def4c6..df4da4d71 100644 --- a/tests/test_legacy.py +++ b/tests/test_legacy.py @@ -91,7 +91,7 @@ class TestPl2004(LegacyTestCase): location = os.path.join(parent_test_dir, 'pl/Tests_2004') normalize = True input_ext = '.text' - exclude = ['Yuri_Footnotes', 'Yuri_Attributes', 'Strong_and_em_together'] + exclude = ['Yuri_Footnotes', 'Yuri_Attributes'] class TestPl2007(LegacyTestCase): @@ -116,8 +116,7 @@ class TestPl2007(LegacyTestCase): 'Code_Blocks', 'Links,_reference_style', 'Backslash_escapes', - 'Code_Spans', - 'Strong_and_em_together' + 'Code_Spans' ] diff --git a/tests/test_syntax/blocks/test_hr.py b/tests/test_syntax/blocks/test_hr.py index 5aa48465b..a8753977b 100644 --- a/tests/test_syntax/blocks/test_hr.py +++ b/tests/test_syntax/blocks/test_hr.py @@ -318,7 +318,7 @@ def test_hr_after_emstrong(self): ), self.dedent( """ -

text

+

text

""" ) diff --git a/tests/test_syntax/extensions/test_legacy_em.py b/tests/test_syntax/extensions/test_legacy_em.py index a1a9f2e5a..ad2862e5e 100644 --- a/tests/test_syntax/extensions/test_legacy_em.py +++ b/tests/test_syntax/extensions/test_legacy_em.py @@ -55,12 +55,12 @@ def test_complex_multple_underscore_type(self): self.assertMarkdownRenders( 'traced ___along___ bla __blocked__ if other ___or___', - '

traced along bla blocked if other or

' # noqa: E501 + '

traced along bla blocked if other or

' # noqa: E501 ) def test_complex_multple_underscore_type_variant2(self): self.assertMarkdownRenders( 'on the __1-4 row__ of the AP Combat Table ___and___ receive', - '

on the 1-4 row of the AP Combat Table and receive

' + '

on the 1-4 row of the AP Combat Table and receive

' ) diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 127d2c78f..f50d35fe7 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -161,14 +161,14 @@ def test_complex_multple_emphasis_type(self): self.assertMarkdownRenders( 'traced ***along*** bla **blocked** if other ***or***', - '

traced along bla blocked if other or

' # noqa: E501 + '

traced along bla blocked if other or

' # noqa: E501 ) def test_complex_multple_emphasis_type_variant2(self): self.assertMarkdownRenders( 'on the **1-4 row** of the AP Combat Table ***and*** receive', - '

on the 1-4 row of the AP Combat Table and receive

' + '

on the 1-4 row of the AP Combat Table and receive

' ) def test_link_emphasis_outer(self): @@ -244,13 +244,13 @@ def test_nested_patterns(self): ), self.dedent( """ -

link - link +

link + link link link link link - link

+ link

I am italic and bold I am just bold

Example bold italic on the same line bold italic.

Example bold italic on the same line bold italic.

@@ -296,7 +296,7 @@ def test_em_strong_complex(self): """

test test test test

test test test test

-

test

+

test

test

test test_

test test

@@ -304,7 +304,7 @@ def test_em_strong_complex(self):

test test test test

test test test test

*test

-

test

+

test

test*

test test

testtest testtest

@@ -328,10 +328,10 @@ def test_strong_and_em_together(self): ), self.dedent( """ -

This is strong and em.

-

So is this word.

-

This is strong and em.

-

So is this word.

+

This is strong and em.

+

So is this word.

+

This is strong and em.

+

So is this word.

""" ) ) @@ -381,16 +381,16 @@ def test_advanced_nesting(self):

abcdef

abcdef

abcdefgh

-

abcde

+

abcde

abcde**f

abcde**f

abcd*ef

_a b c

_a __b __c __d __e _f

-

a b c d e f

-

a b c d e f

-

a b c d e f g h

-

a b c d e

+

a b c d e f

+

a b c d e f

+

a b c d e f g h

+

a b c d e

_a b _c d _e f

""" ) @@ -549,6 +549,7 @@ def test_commonmark(self): *foo**bar*** + foo***bar***baz foo******bar*********baz @@ -653,6 +654,7 @@ def test_commonmark(self): ******foo****** + ***foo*** _____foo_____ @@ -767,7 +769,8 @@ def test_commonmark(self):

foo bar

foo bar

foobar

-

foobarbaz

+ +

foobarbaz

foobar***baz

foo bar baz bim bop

foo bar

@@ -820,8 +823,9 @@ def test_commonmark(self):

foo

foo

foo

-

foo

-

foo

+ +

foo

+

foo

foo _bar baz_

From 8a12d536facb59fcfd93ea7740ee2501208fd980 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Sun, 13 Sep 2026 06:58:40 -0600 Subject: [PATCH 24/34] Forgot to commit last fix, better cache point Ensure the cache point is after the last region start, but before or equal to the next region start so we can properly adjust the offset when returning the next region from the cache. --- markdown/inlinepatterns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 4eb0ec1a7..15b43aefc 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -754,7 +754,7 @@ def increment_next_position(self, start: int, count: int) -> None: self.cache_pos = self.regions[self.cache_index][0] while self.stack: entry = self.stack.popleft() - if entry[0] > start: + if start < entry[0] <= self.cache_pos: self.cache_pos = entry[0] break From 606fc24f480a0a76863dcc6e1ba9e278514b255d Mon Sep 17 00:00:00 2001 From: facelessuser Date: Sun, 13 Sep 2026 21:09:52 -0600 Subject: [PATCH 25/34] Another fix that missed getting committed If the final delimiter we are comparing against is greater than 2, consume it anyway. --- markdown/inlinepatterns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 15b43aefc..3905d6166 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -882,7 +882,7 @@ def handleMatch( # type: ignore[override] while stack and delimiter[-1] != 3 and delimiter[2]: delimiter = stack.pop() last = delimiter[-1] - if delimiter[2]: + if delimiter[2] and delimiter[-1] != 3: break is_start = False From 0901c7889314530563c76b26cdc4841e690c8f74 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Mon, 14 Sep 2026 15:08:48 -0600 Subject: [PATCH 26/34] Combine emphasis handling into one processor DelimiterProcessor can now handle multiple delimiters at the same time allowing for more natural nesting. If the `em_strong` inline processor already exists, tokens are added to the existing processor; otherwise, a new delimiter processor is added. --- markdown/core.py | 11 +- markdown/extensions/legacy_em.py | 7 +- markdown/inlinepatterns.py | 302 ++++++++++++++-------- tests/test_syntax/inline/test_emphasis.py | 26 +- 4 files changed, 217 insertions(+), 129 deletions(-) diff --git a/markdown/core.py b/markdown/core.py index a04510c03..a62b31160 100644 --- a/markdown/core.py +++ b/markdown/core.py @@ -119,7 +119,7 @@ def __init__(self, **kwargs): self.registeredExtensions: list[Extension] = [] self.docType = "" # TODO: Maybe delete this. It does not appear to be used anymore. self.stripTopLevelTags: bool = True - self.delimiters: dict[str, DelimiterProcessor] = {} + self.delimiters: DelimiterProcessor | None = None self.build_parser() @@ -272,11 +272,10 @@ def reset(self) -> Markdown: self.htmlStash.reset() self.references.clear() - for key in list(self.delimiters): - ext = self.delimiters[key] - ext.reset() - if ext not in self.inlinePatterns: - del self.delimiters[key] + if self.delimiters is not None: + self.delimiters.reset() + if self.delimiters not in self.inlinePatterns: + self.delimiters = None for extension in self.registeredExtensions: if hasattr(extension, 'reset'): diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index e0edf05b4..4ce47176e 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -25,12 +25,15 @@ def extendMarkdown(self, md): | Class Instance | Registry | Name | Priority | | ------------------------------------------------------------------ | ---------------------------------------------------------------- | ------------ | :------: | - | [`DelimiterProcessor`][markdown.inlinepatterns.DelimiterProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong2` | `50` | + | [`DelimiterProcessor`][markdown.inlinepatterns.DelimiterProcessor] | [`inlinepatterns`][markdown.inlinepatterns.build_inlinepatterns] | `em_strong` | `60` | """ # flake8: noqa: E501 27-29 - md.inlinePatterns.register(DelimiterProcessor(r'_', 'strong,em', md), 'em_strong2', 50) + if md.delimiters is not None: + md.delimiters.add('_', 'strong,em') + else: + md.inlinePatterns.register(DelimiterProcessor('_', 'strong,em', md), 'em_strong', 60) def makeExtension(**kwargs): # pragma: no cover """ Return an instance of the `LegacyEmExtension` """ diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 3905d6166..788bba1f7 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -91,7 +91,7 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) inlinePatterns.register(DelimiterProcessor('*', 'strong,em', md), 'em_strong', 60) - inlinePatterns.register(DelimiterProcessor('_', 'strong,em', md, smart=True), 'em_strong2', 50) + md.delimiters.add('_', 'strong,em', smart=True) return inlinePatterns @@ -563,50 +563,20 @@ def _unescape(m: re.Match[str]) -> str: return RE.sub(_unescape, text) -class DelimiterProcessor(InlineProcessor): - """Processor for handling complex nested patterns such as strong and em matches.""" - - def __init__( - self, - token: str, - tags: str, - md: Markdown, - smart: bool = False, - double: bool = False - ) -> None: - """ - Initialize. +class Delimiter: + """Delimiter.""" - Arguments: - token: A single character token. - tags: A tag or two tags seprated by comma. When two are specified, the first will be the - one that takes double tokens. - md: the Markdown object - smart: Enable intelligent word logic. - double: If only one tag is specified, indicate whether it requires double tokens. - - """ + def __init__(self, token: str, tags: str, smart: bool, double: bool): + """Initialize.""" - # Cache info - md.delimiters[token] = self - self.regions: list[tuple[int, int, int, int, int]] = [] self.stack: deque[tuple[int, int, bool, int]] = deque() - self.cache_index = 0 - self.cache_pos = 0 - - self.last_run = 0.0 + temp = tags.split(',') + self.tag_count = len(temp) + self.tags: tuple[str, str] = (temp[0], temp[1]) if self.tag_count == 2 else (temp[0], temp[0]) + self.double = len(temp) != 2 and double + self.single = len(temp) != 2 and not double self.smart = smart - self.tags = tags.split(',') - self.double = len(self.tags) != 2 and double - super().__init__(self._build_patterns(token), md) - - def reset(self) -> None: - """Rest.""" - - self.regions.clear() - self.stack.clear() - self.cache_index = 0 - self.cache_pos = 0 + self._build_patterns(token) def _build_patterns(self, token: str) -> str: """Build regular expression patterns.""" @@ -614,35 +584,108 @@ def _build_patterns(self, token: str) -> str: # Build up patterns self.token = token etoken = re.escape(token) - avoid_start = fr'(?:(?<=_)|(?(?(?{avoid_start}{etoken}{{1,}}(?![\s{etoken}])(?!$)) + (?P(?(?{xstart}{etoken}{{1,}}(?![\s{etoken}])(?!$)) ''', flags=re.UNICODE ) # Patterns for "dumb" cases. else: self.boundary = re.compile( - fr'''(?x)(?: + fr'''(?x) (?P(?(?{etoken}{{1,}}(?![\s{etoken}])(?!$)) - )''', + ''', flags=re.UNICODE ) return fr'{etoken}' + def reset(self) -> None: + """Reset.""" + + # Cache info + self.stack.clear() + + +class DelimiterProcessor(InlineProcessor): + """Processor for handling complex nested patterns such as strong and em matches.""" + + SPACE = re.compile(r'\s') + + def __init__( + self, + token: str, + tags: str, + md: Markdown, + smart: bool = False, + double: bool = False + ) -> None: + """Initialize.""" + + md.delimiters = self + self.regions: list[tuple[int, int, int, int, tuple[str, str], int]] = [] + self.stack: list[tuple[int, int, bool, int]] = [] + self.tokens: list[str] = [] + self.delimiters: dict[str, Delimiter] = {} + self.cache_index = 0 + self.cache_pos = 0 + self.md = md + # API for Markdown to pass `safe_mode` into instance + self.safe_mode = False + self.add(token, tags, smart, double) + + def add( + self, + token: str, + tags: str, + smart: bool = False, + double: bool = False + ) -> None: + """Add a delimiter.""" + + if token not in self.tokens: + self.tokens.append(token) + self.delimiters[token] = Delimiter(token, tags, smart, double) + self.pattern = '|'.join([re.escape(t) for t in self.tokens]) + self.compiled_re = re.compile(self.pattern, re.DOTALL | re.UNICODE) + + def remove(self, token: str) -> None: + """Remove a token.""" + + try: + i = self.tokens.index(token) + del self.tokens[i] + del self.delimiters[token] + except Exception: + pass + self.pattern = '|'.join([re.escape(t) for t in self.tokens]) + self.compiled_re = re.compile(self.pattern, re.DOTALL | re.UNICODE) if self.pattern else re.compile(r'(?!)') + + def reset(self) -> None: + """Reset.""" + + # Cache info + for v in self.delimiters.values(): + v.reset() + self.regions.clear() + self.stack.clear() + self.cache_index = 0 + self.cache_pos = 0 + def _build_element( self, data: str, @@ -655,20 +698,9 @@ def _build_element( el: etree.Element | None = None last: Any = None previous: Any = None - greater: Any = None - lesser: Any = None outer: list[etree.Element] = [] - outer_r: list[tuple[int, int, int, int, int]] = [] - - if len(self.tags) == 2: - greater, lesser = self.tags - elif self.double: # pragma: no cover - greater = self.tags[0] - lesser = None - else: # pragma: no cover - lesser = self.tags[0] - greater = None + outer_r: list[tuple[int, int, int, int, tuple[str, str], int]] = [] # Iterate regions creating the elements they represent end = len(regions) @@ -679,11 +711,12 @@ def _build_element( if idx and r[0] >= regions[start][3]: idx -= 1 break + # Get the appropriate element(s) - if r[4] == 2: - el1 = etree.Element(greater) + if r[-1] == 2: + el1 = etree.Element(r[4][0]) else: - el1 = etree.Element(lesser) + el1 = etree.Element(r[4][1]) # Populate the elements with their text if idx > 1: @@ -753,7 +786,7 @@ def increment_next_position(self, start: int, count: int) -> None: if self.cache_index < len(self.regions): self.cache_pos = self.regions[self.cache_index][0] while self.stack: - entry = self.stack.popleft() + entry = self.stack.pop(0) if start < entry[0] <= self.cache_pos: self.cache_pos = entry[0] break @@ -773,6 +806,40 @@ def get_cached_result(self, pos: int, data: str) -> tuple[etree.Element, int, in self.increment_next_position(start, count) return el, start + offset, end + offset + def get_match(self, data: str, start: int) -> re.Match[str] | None: + """Get match.""" + + for d in self.delimiters.values(): + m = d.boundary.match(data, start) + if m is not None: + return m + return None + + def search(self, data: str, start: int) -> re.Match[str] | None: + """Search.""" + + for i in range(start, len(data)): + if data[i] in self.delimiters: + d = self.delimiters[data[i]] + m = d.boundary.match(data, i) + if m is not None: + if not d.stack and m.lastgroup[0] == 'e': # type: ignore[index] + continue + return m + return None + + def add_region(self, delim: Delimiter, a: int, b: int, c: int, d: int, size: int) -> None: + """Add region.""" + + self.regions.append((a, b, c, d, delim.tags, size)) + for delim in self.delimiters.values(): + while delim.stack: + p = delim.stack[-1][0] + if max(p, a) <= min(p, d - 1): + delim.stack.pop() + continue + break + def handleMatch( # type: ignore[override] self, m: re.Match[str], @@ -785,7 +852,7 @@ def handleMatch( # type: ignore[override] return self.get_cached_result(m.start(0), data) # If token is not an opening, quit - m2 = self.boundary.match(data, m.start(0)) + m2 = self.get_match(data, m.start(0)) if m2 is None or m2.lastgroup[0] == 'e': # type: ignore[index] if m2 is not None: m = m2 @@ -793,37 +860,44 @@ def handleMatch( # type: ignore[override] return None, m.start(0), m.end(0) # Get the stack and regions - stack = self.stack - regions = self.regions + token = data[m.start(0)] + delim = self.delimiters[token] + stack = delim.stack start = m2.start(0) end = m2.end(0) length = end - start # Double needs at least a size of 2 - if self.double and length < 2: + if delim.double and length < 2: return None, m.start(0), m.end(0) is_ambiguous = m2.lastgroup[0] != 's' # type: ignore[index] - self.stack.append((start, start + length, is_ambiguous, length)) + stack.append((start, start + length, is_ambiguous, length)) # Pair tokens until the stack is empty or we can no longer find tokens. - while stack: - m2 = self.boundary.search(data, end) + while any(d.stack for d in self.delimiters.values()): + m2 = self.search(data, end) if m2 is None: break + + token = data[m2.start(0)] + delim = self.delimiters[token] + stack = delim.stack + start = m2.start(0) end = m2.end(0) # Get current and last delimiter size current = len(m2.group(0)) - last = stack[-1][-1] # Some delimiters may be ambiguous and look like both a start or an end is_start = m2.lastgroup[0] != 'e' # type: ignore[index] is_end = not is_start or m2.lastgroup[0] != 's' # type: ignore[index] is_ambiguous = is_start and is_end + last = stack[-1][-1] if stack else 0 + # Find closing tokens # Looking for: # - `*em*` @@ -836,35 +910,36 @@ def handleMatch( # type: ignore[override] # Avoid ambiguous tokens that could be a start or an end. # Consume starts until the end token is fully consumed. # If we don't consume the entire end, see if next rule consumes it. - if ( - is_end and - ((not is_ambiguous and current > last) or current == last or current >= 3) - ): + if stack and is_end and ((not is_ambiguous and current > last) or current == last or current >= 3): is_start = False - # Consume previous tokens until the delimiter is consumed + # Consume previous points until the delimiter is consumed original = current - while current and last <= current: + while stack and current and last <= current: delimiter = stack.pop() # Build up region for pair and adjust accounting. - # Special handle span of 3 to preserve old Python Markdown behavior. - # For true CommonMark logic `2` should always be used. - size = min(delimiter[-1], 1 if len(self.tags) == 2 and delimiter[-1] == 3 else self.max_size) - regions.append((delimiter[1] - size, delimiter[1], start, start + size, size)) + size = min(delimiter[-1], 1 if delim.tag_count == 2 and delimiter[-1] == 3 else delim.max_size) + self.add_region(delim, delimiter[1] - size, delimiter[1], start, start + size, size) start += size current -= size - if size < delimiter[-1] and (not self.double or (delimiter[-1] - size) != 1): + new = 0 + if size < delimiter[-1] and (not delim.double or (delimiter[-1] - size) != 1): new = delimiter[-1] - size stack.append((delimiter[0], delimiter[1] - size, delimiter[2], new)) + if not stack: + if any(d.stack for d in self.delimiters.values() if d is not delim): + delim.reset() + continue is_end = False break + last = stack[-1][-1] # Should remainder be treated as a new start? if original >= 3 and current and is_ambiguous: - self.stack.append((regions[-1][3], end, False, current)) + delim.stack.append((m2.start(0) + (original - current), end, False, current)) is_end = False # Do we still have more to consume? @@ -875,49 +950,62 @@ def handleMatch( # type: ignore[override] # - `***em*` # - `***strong**` # - `**em*` - if is_end and (last >= 3 or not is_ambiguous) and last > current: + if stack and is_end and (last >= 3 or not is_ambiguous) and last > current: delimiter = stack.pop() # Don't pair with an ambiguous opening while stack and delimiter[-1] != 3 and delimiter[2]: delimiter = stack.pop() last = delimiter[-1] + if delimiter[2] and delimiter[-1] != 3: + if any(d.stack for d in self.delimiters.values() if d is not delim): + delim.reset() + continue break - is_start = False - ds, de = delimiter[:2] - while current and (not self.double or current != 1): - size = min(current, 1 if len(self.tags) == 2 and current == 3 else self.max_size) - new = last - size - regions.append((ds + new, de, start, start + size, size)) - start += size - current -= size - last -= size - de -= size - if not self.double or last > 1: - stack.append((ds, de, False, last)) + ignore = False + # Create new region if end is valid. + # If not valid, ignore the end but continue parsing. + if not ignore: + is_start = False + ds, de = delimiter[:2] + while current and (not delim.double or current != 1): + size = min(current, 1 if delim.tag_count == 2 and current == 3 else delim.max_size) + new = last - size + self.add_region(delim, ds + new, de, start, start + size, size) + start += size + current -= size + last -= size + de -= size + if not delim.double or last > 1: + stack.append((ds, de, False, last)) # Find opening tokens - if is_start and (not self.double or current != 1): - # Looking for: - # - `*em ...*` - # - `**strong ...*` - # - `***em ...*` + # Looking for: + # - `*em ...*` + # - `**strong ...*` + # - `***em ...*` + if is_start and (not delim.double or current != 1): stack.append((start, end, is_ambiguous, current)) + # Combine the stacks and order them + for delim in self.delimiters.values(): + self.stack.extend(delim.stack) + self.stack.sort(key=lambda x: x[0]) + # Build the HTML elements - if regions: + if self.regions: # Regions may be out of order. - regions.sort(key=lambda x: x[0]) - start, end = regions[0][0], regions[0][3] + self.regions.sort(key=lambda x: x[0]) + start, end = self.regions[0][0], self.regions[0][3] el, count = self._build_element(data) self.increment_next_position(start, count) return el, start, end # We failed to pair any valid start/end delimiters, avoid the parsed range next pass. start = m.start(0) - end = stack[-1][1] if stack else m.end(0) + end = self.stack[-1][1] if self.stack else m.end(0) self.reset() return None, start, end diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index f50d35fe7..a5c948ef6 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -372,6 +372,10 @@ def test_advanced_nesting(self): ___a ___b c__ d_ e___ _a __b__ _c __d__ _e __f__ + + ___bold and italic***bold and italic**bold and italic*__ italic_ + + ***a __b** __c *d__ e* """ ), self.dedent( @@ -392,7 +396,9 @@ def test_advanced_nesting(self):

a b c d e f g h

a b c d e

_a b _c d _e f

- """ +

bold and italicbold and italicbold and italic italic

+

a __b c *d e

+ """ # noqa: E501 ) ) @@ -661,8 +667,7 @@ def test_commonmark(self): *foo _bar* baz_ - + *foo __bar *baz bim__ bam* **foo **bar baz** @@ -827,9 +832,7 @@ def test_commonmark(self):

foo

foo

foo _bar baz_

- - +

foo bar *baz bim bam

**foo bar baz

*foo bar baz

*bar*

@@ -856,17 +859,12 @@ def test_remove_processor(self): # Remove all delimiter processors md = markdown.Markdown() - extensions = md.delimiters.values() - self.assertEqual(len(extensions), 2) - - for ext in md.delimiters.values(): - self.assertTrue(isinstance(ext, DelimiterProcessor)) + self.assertTrue(md.delimiters is not None) + self.assertTrue(isinstance(md.delimiters, DelimiterProcessor)) md.inlinePatterns.deregister('em_strong') - md.inlinePatterns.deregister('em_strong2') # Call reset which will cause them to remove themselves from being registered md.reset() - extensions = md.delimiters.values() - self.assertEqual(len(extensions), 0) + self.assertTrue(md.delimiters is None) From 572da3542b796fb32421318698eff2a72cab453b Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 15 Sep 2026 13:12:09 -0600 Subject: [PATCH 27/34] Add punctuation logic --- markdown/inlinepatterns.py | 111 ++++++++++++++++++++-- tests/test_syntax/inline/test_emphasis.py | 39 ++++---- 2 files changed, 121 insertions(+), 29 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 788bba1f7..1a5f4342e 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -563,6 +563,83 @@ def _unescape(m: re.Match[str]) -> str: return RE.sub(_unescape, text) +PUNCT = ( + b'!-/:-@\\[-`{-\\~\xc2\xa1-\xc2\xa9\xc2\xab-\xc2\xac\xc2\xae-\xc2\xb1\xc2\xb4\xc2\xb6-\xc2\xb8\xc2\xbb\xc2\xbf\xc3' + b'\x97\xc3\xb7\xcb\x82-\xcb\x85\xcb\x92-\xcb\x9f\xcb\xa5-\xcb\xab\xcb\xad\xcb\xaf-\xcb\xbf\xcd\xb5\xcd\xbe\xce\x84' + b'-\xce\x85\xce\x87\xcf\xb6\xd2\x82\xd5\x9a-\xd5\x9f\xd6\x89-\xd6\x8a\xd6\x8d-\xd6\x8f\xd6\xbe\xd7\x80\xd7\x83\xd7' + b'\x86\xd7\xb3-\xd7\xb4\xd8\x86-\xd8\x8f\xd8\x9b\xd8\x9d-\xd8\x9f\xd9\xaa-\xd9\xad\xdb\x94\xdb\x9e\xdb\xa9\xdb\xbd' + b'-\xdb\xbe\xdc\x80-\xdc\x8d\xdf\xb6-\xdf\xb9\xdf\xbe-\xdf\xbf\xe0\xa0\xb0-\xe0\xa0\xbe\xe0\xa1\x9e\xe0\xa2\x88' + b'\xe0\xa5\xa4-\xe0\xa5\xa5\xe0\xa5\xb0\xe0\xa7\xb2-\xe0\xa7\xb3\xe0\xa7\xba-\xe0\xa7\xbb\xe0\xa7\xbd\xe0\xa9\xb6' + b'\xe0\xab\xb0-\xe0\xab\xb1\xe0\xad\xb0\xe0\xaf\xb3-\xe0\xaf\xba\xe0\xb1\xb7\xe0\xb1\xbf\xe0\xb2\x84\xe0\xb5\x8f' + b'\xe0\xb5\xb9\xe0\xb7\xb4\xe0\xb8\xbf\xe0\xb9\x8f\xe0\xb9\x9a-\xe0\xb9\x9b\xe0\xbc\x81-\xe0\xbc\x97\xe0\xbc\x9a-' + b'\xe0\xbc\x9f\xe0\xbc\xb4\xe0\xbc\xb6\xe0\xbc\xb8\xe0\xbc\xba-\xe0\xbc\xbd\xe0\xbe\x85\xe0\xbe\xbe-\xe0\xbf\x85' + b'\xe0\xbf\x87-\xe0\xbf\x8c\xe0\xbf\x8e-\xe0\xbf\x9a\xe1\x81\x8a-\xe1\x81\x8f\xe1\x82\x9e-\xe1\x82\x9f\xe1\x83\xbb' + b'\xe1\x8d\xa0-\xe1\x8d\xa8\xe1\x8e\x90-\xe1\x8e\x99\xe1\x90\x80\xe1\x99\xad-\xe1\x99\xae\xe1\x9a\x9b-\xe1\x9a\x9c' + b'\xe1\x9b\xab-\xe1\x9b\xad\xe1\x9c\xb5-\xe1\x9c\xb6\xe1\x9f\x94-\xe1\x9f\x96\xe1\x9f\x98-\xe1\x9f\x9b\xe1\xa0\x80' + b'-\xe1\xa0\x8a\xe1\xa5\x80\xe1\xa5\x84-\xe1\xa5\x85\xe1\xa7\x9e-\xe1\xa7\xbf\xe1\xa8\x9e-\xe1\xa8\x9f\xe1\xaa\xa0' + b'-\xe1\xaa\xa6\xe1\xaa\xa8-\xe1\xaa\xad\xe1\xad\x8e-\xe1\xad\x8f\xe1\xad\x9a-\xe1\xad\xaa\xe1\xad\xb4-\xe1\xad' + b'\xbf\xe1\xaf\xbc-\xe1\xaf\xbf\xe1\xb0\xbb-\xe1\xb0\xbf\xe1\xb1\xbe-\xe1\xb1\xbf\xe1\xb3\x80-\xe1\xb3\x87\xe1\xb3' + b'\x93\xe1\xbe\xbd\xe1\xbe\xbf-\xe1\xbf\x81\xe1\xbf\x8d-\xe1\xbf\x8f\xe1\xbf\x9d-\xe1\xbf\x9f\xe1\xbf\xad-\xe1\xbf' + b'\xaf\xe1\xbf\xbd-\xe1\xbf\xbe\xe2\x80\x90-\xe2\x80\xa7\xe2\x80\xb0-\xe2\x81\x9e\xe2\x81\xba-\xe2\x81\xbe\xe2\x82' + b'\x8a-\xe2\x82\x8e\xe2\x82\xa0-\xe2\x83\x80\xe2\x84\x80-\xe2\x84\x81\xe2\x84\x83-\xe2\x84\x86\xe2\x84\x88-\xe2' + b'\x84\x89\xe2\x84\x94\xe2\x84\x96-\xe2\x84\x98\xe2\x84\x9e-\xe2\x84\xa3\xe2\x84\xa5\xe2\x84\xa7\xe2\x84\xa9\xe2' + b'\x84\xae\xe2\x84\xba-\xe2\x84\xbb\xe2\x85\x80-\xe2\x85\x84\xe2\x85\x8a-\xe2\x85\x8d\xe2\x85\x8f\xe2\x86\x8a-\xe2' + b'\x86\x8b\xe2\x86\x90-\xe2\x90\xa9\xe2\x91\x80-\xe2\x91\x8a\xe2\x92\x9c-\xe2\x93\xa9\xe2\x94\x80-\xe2\x9d\xb5\xe2' + b'\x9e\x94-\xe2\xad\xb3\xe2\xad\xb6-\xe2\xae\x95\xe2\xae\x97-\xe2\xaf\xbf\xe2\xb3\xa5-\xe2\xb3\xaa\xe2\xb3\xb9-' + b'\xe2\xb3\xbc\xe2\xb3\xbe-\xe2\xb3\xbf\xe2\xb5\xb0\xe2\xb8\x80-\xe2\xb8\xae\xe2\xb8\xb0-\xe2\xb9\x9d\xe2\xba\x80-' + b'\xe2\xba\x99\xe2\xba\x9b-\xe2\xbb\xb3\xe2\xbc\x80-\xe2\xbf\x95\xe2\xbf\xb0-\xe2\xbf\xbf\xe3\x80\x81-\xe3\x80\x84' + b'\xe3\x80\x88-\xe3\x80\xa0\xe3\x80\xb0\xe3\x80\xb6-\xe3\x80\xb7\xe3\x80\xbd-\xe3\x80\xbf\xe3\x82\x9b-\xe3\x82\x9c' + b'\xe3\x82\xa0\xe3\x83\xbb\xe3\x86\x90-\xe3\x86\x91\xe3\x86\x96-\xe3\x86\x9f\xe3\x87\x80-\xe3\x87\xa5\xe3\x87\xaf' + b'\xe3\x88\x80-\xe3\x88\x9e\xe3\x88\xaa-\xe3\x89\x87\xe3\x89\x90\xe3\x89\xa0-\xe3\x89\xbf\xe3\x8a\x8a-\xe3\x8a\xb0' + b'\xe3\x8b\x80-\xe3\x8f\xbf\xe4\xb7\x80-\xe4\xb7\xbf\xea\x92\x90-\xea\x93\x86\xea\x93\xbe-\xea\x93\xbf\xea\x98\x8d' + b'-\xea\x98\x8f\xea\x99\xb3\xea\x99\xbe\xea\x9b\xb2-\xea\x9b\xb7\xea\x9c\x80-\xea\x9c\x96\xea\x9c\xa0-\xea\x9c\xa1' + b'\xea\x9e\x89-\xea\x9e\x8a\xea\xa0\xa8-\xea\xa0\xab\xea\xa0\xb6-\xea\xa0\xb9\xea\xa1\xb4-\xea\xa1\xb7\xea\xa3\x8e' + b'-\xea\xa3\x8f\xea\xa3\xb8-\xea\xa3\xba\xea\xa3\xbc\xea\xa4\xae-\xea\xa4\xaf\xea\xa5\x9f\xea\xa7\x81-\xea\xa7\x8d' + b'\xea\xa7\x9e-\xea\xa7\x9f\xea\xa9\x9c-\xea\xa9\x9f\xea\xa9\xb7-\xea\xa9\xb9\xea\xab\x9e-\xea\xab\x9f\xea\xab\xb0' + b'-\xea\xab\xb1\xea\xad\x9b\xea\xad\xaa-\xea\xad\xab\xea\xaf\xab\xef\xac\xa9\xef\xae\xb2-\xef\xaf\x82\xef\xb4\xbe-' + b'\xef\xb5\x8f\xef\xb7\x8f\xef\xb7\xbc-\xef\xb7\xbf\xef\xb8\x90-\xef\xb8\x99\xef\xb8\xb0-\xef\xb9\x92\xef\xb9\x94-' + b'\xef\xb9\xa6\xef\xb9\xa8-\xef\xb9\xab\xef\xbc\x81-\xef\xbc\x8f\xef\xbc\x9a-\xef\xbc\xa0\xef\xbc\xbb-\xef\xbd\x80' + b'\xef\xbd\x9b-\xef\xbd\xa5\xef\xbf\xa0-\xef\xbf\xa6\xef\xbf\xa8-\xef\xbf\xae\xef\xbf\xbc-\xef\xbf\xbd\xf0\x90\x84' + b'\x80-\xf0\x90\x84\x82\xf0\x90\x84\xb7-\xf0\x90\x84\xbf\xf0\x90\x85\xb9-\xf0\x90\x86\x89\xf0\x90\x86\x8c-\xf0\x90' + b'\x86\x8e\xf0\x90\x86\x90-\xf0\x90\x86\x9c\xf0\x90\x86\xa0\xf0\x90\x87\x90-\xf0\x90\x87\xbc\xf0\x90\x8e\x9f\xf0' + b'\x90\x8f\x90\xf0\x90\x95\xaf\xf0\x90\xa1\x97\xf0\x90\xa1\xb7-\xf0\x90\xa1\xb8\xf0\x90\xa4\x9f\xf0\x90\xa4\xbf' + b'\xf0\x90\xa9\x90-\xf0\x90\xa9\x98\xf0\x90\xa9\xbf\xf0\x90\xab\x88\xf0\x90\xab\xb0-\xf0\x90\xab\xb6\xf0\x90\xac' + b'\xb9-\xf0\x90\xac\xbf\xf0\x90\xae\x99-\xf0\x90\xae\x9c\xf0\x90\xb5\xae\xf0\x90\xb6\x8e-\xf0\x90\xb6\x8f\xf0\x90' + b'\xba\xad\xf0\x90\xbd\x95-\xf0\x90\xbd\x99\xf0\x90\xbe\x86-\xf0\x90\xbe\x89\xf0\x91\x81\x87-\xf0\x91\x81\x8d\xf0' + b'\x91\x82\xbb-\xf0\x91\x82\xbc\xf0\x91\x82\xbe-\xf0\x91\x83\x81\xf0\x91\x85\x80-\xf0\x91\x85\x83\xf0\x91\x85\xb4-' + b'\xf0\x91\x85\xb5\xf0\x91\x87\x85-\xf0\x91\x87\x88\xf0\x91\x87\x8d\xf0\x91\x87\x9b\xf0\x91\x87\x9d-\xf0\x91\x87' + b'\x9f\xf0\x91\x88\xb8-\xf0\x91\x88\xbd\xf0\x91\x8a\xa9\xf0\x91\x8f\x94-\xf0\x91\x8f\x95\xf0\x91\x8f\x97-\xf0\x91' + b'\x8f\x98\xf0\x91\x91\x8b-\xf0\x91\x91\x8f\xf0\x91\x91\x9a-\xf0\x91\x91\x9b\xf0\x91\x91\x9d\xf0\x91\x93\x86\xf0' + b'\x91\x97\x81-\xf0\x91\x97\x97\xf0\x91\x99\x81-\xf0\x91\x99\x83\xf0\x91\x99\xa0-\xf0\x91\x99\xac\xf0\x91\x9a\xb9' + b'\xf0\x91\x9c\xbc-\xf0\x91\x9c\xbf\xf0\x91\xa0\xbb\xf0\x91\xa5\x84-\xf0\x91\xa5\x86\xf0\x91\xa7\xa2\xf0\x91\xa8' + b'\xbf-\xf0\x91\xa9\x86\xf0\x91\xaa\x9a-\xf0\x91\xaa\x9c\xf0\x91\xaa\x9e-\xf0\x91\xaa\xa2\xf0\x91\xac\x80-\xf0\x91' + b'\xac\x89\xf0\x91\xaf\xa1\xf0\x91\xb1\x81-\xf0\x91\xb1\x85\xf0\x91\xb1\xb0-\xf0\x91\xb1\xb1\xf0\x91\xbb\xb7-\xf0' + b'\x91\xbb\xb8\xf0\x91\xbd\x83-\xf0\x91\xbd\x8f\xf0\x91\xbf\x95-\xf0\x91\xbf\xb1\xf0\x91\xbf\xbf\xf0\x92\x91\xb0-' + b'\xf0\x92\x91\xb4\xf0\x92\xbf\xb1-\xf0\x92\xbf\xb2\xf0\x96\xa9\xae-\xf0\x96\xa9\xaf\xf0\x96\xab\xb5\xf0\x96\xac' + b'\xb7-\xf0\x96\xac\xbf\xf0\x96\xad\x84-\xf0\x96\xad\x85\xf0\x96\xb5\xad-\xf0\x96\xb5\xaf\xf0\x96\xba\x97-\xf0\x96' + b'\xba\x9a\xf0\x96\xbf\xa2\xf0\x9b\xb2\x9c\xf0\x9b\xb2\x9f\xf0\x9c\xb0\x80-\xf0\x9c\xb3\xaf\xf0\x9c\xb4\x80-\xf0' + b'\x9c\xba\xb3\xf0\x9c\xbd\x90-\xf0\x9c\xbf\x83\xf0\x9d\x80\x80-\xf0\x9d\x83\xb5\xf0\x9d\x84\x80-\xf0\x9d\x84\xa6' + b'\xf0\x9d\x84\xa9-\xf0\x9d\x85\xa4\xf0\x9d\x85\xaa-\xf0\x9d\x85\xac\xf0\x9d\x86\x83-\xf0\x9d\x86\x84\xf0\x9d\x86' + b'\x8c-\xf0\x9d\x86\xa9\xf0\x9d\x86\xae-\xf0\x9d\x87\xaa\xf0\x9d\x88\x80-\xf0\x9d\x89\x81\xf0\x9d\x89\x85\xf0\x9d' + b'\x8c\x80-\xf0\x9d\x8d\x96\xf0\x9d\x9b\x81\xf0\x9d\x9b\x9b\xf0\x9d\x9b\xbb\xf0\x9d\x9c\x95\xf0\x9d\x9c\xb5\xf0' + b'\x9d\x9d\x8f\xf0\x9d\x9d\xaf\xf0\x9d\x9e\x89\xf0\x9d\x9e\xa9\xf0\x9d\x9f\x83\xf0\x9d\xa0\x80-\xf0\x9d\xa7\xbf' + b'\xf0\x9d\xa8\xb7-\xf0\x9d\xa8\xba\xf0\x9d\xa9\xad-\xf0\x9d\xa9\xb4\xf0\x9d\xa9\xb6-\xf0\x9d\xaa\x83\xf0\x9d\xaa' + b'\x85-\xf0\x9d\xaa\x8b\xf0\x9e\x85\x8f\xf0\x9e\x8b\xbf\xf0\x9e\x97\xbf\xf0\x9e\xa5\x9e-\xf0\x9e\xa5\x9f\xf0\x9e' + b'\xb2\xac\xf0\x9e\xb2\xb0\xf0\x9e\xb4\xae\xf0\x9e\xbb\xb0-\xf0\x9e\xbb\xb1\xf0\x9f\x80\x80-\xf0\x9f\x80\xab\xf0' + b'\x9f\x80\xb0-\xf0\x9f\x82\x93\xf0\x9f\x82\xa0-\xf0\x9f\x82\xae\xf0\x9f\x82\xb1-\xf0\x9f\x82\xbf\xf0\x9f\x83\x81-' + b'\xf0\x9f\x83\x8f\xf0\x9f\x83\x91-\xf0\x9f\x83\xb5\xf0\x9f\x84\x8d-\xf0\x9f\x86\xad\xf0\x9f\x87\xa6-\xf0\x9f\x88' + b'\x82\xf0\x9f\x88\x90-\xf0\x9f\x88\xbb\xf0\x9f\x89\x80-\xf0\x9f\x89\x88\xf0\x9f\x89\x90-\xf0\x9f\x89\x91\xf0\x9f' + b'\x89\xa0-\xf0\x9f\x89\xa5\xf0\x9f\x8c\x80-\xf0\x9f\x9b\x97\xf0\x9f\x9b\x9c-\xf0\x9f\x9b\xac\xf0\x9f\x9b\xb0-\xf0' + b'\x9f\x9b\xbc\xf0\x9f\x9c\x80-\xf0\x9f\x9d\xb6\xf0\x9f\x9d\xbb-\xf0\x9f\x9f\x99\xf0\x9f\x9f\xa0-\xf0\x9f\x9f\xab' + b'\xf0\x9f\x9f\xb0\xf0\x9f\xa0\x80-\xf0\x9f\xa0\x8b\xf0\x9f\xa0\x90-\xf0\x9f\xa1\x87\xf0\x9f\xa1\x90-\xf0\x9f\xa1' + b'\x99\xf0\x9f\xa1\xa0-\xf0\x9f\xa2\x87\xf0\x9f\xa2\x90-\xf0\x9f\xa2\xad\xf0\x9f\xa2\xb0-\xf0\x9f\xa2\xbb\xf0\x9f' + b'\xa3\x80-\xf0\x9f\xa3\x81\xf0\x9f\xa4\x80-\xf0\x9f\xa9\x93\xf0\x9f\xa9\xa0-\xf0\x9f\xa9\xad\xf0\x9f\xa9\xb0-\xf0' + b'\x9f\xa9\xbc\xf0\x9f\xaa\x80-\xf0\x9f\xaa\x89\xf0\x9f\xaa\x8f-\xf0\x9f\xab\x86\xf0\x9f\xab\x8e-\xf0\x9f\xab\x9c' + b'\xf0\x9f\xab\x9f-\xf0\x9f\xab\xa9\xf0\x9f\xab\xb0-\xf0\x9f\xab\xb8\xf0\x9f\xac\x80-\xf0\x9f\xae\x92\xf0\x9f\xae' + b'\x94-\xf0\x9f\xaf\xaf' +).decode('utf-8') + + class Delimiter: """Delimiter.""" @@ -591,13 +668,26 @@ def _build_patterns(self, token: str) -> str: if self.tag_count != 2 and not self.double: self.max_size = 1 + # Python Markdown uses `STX` and `ETX` for placeholders. + # Include handling for these characters in addition to CommonMark rules. + stx, etx = '\x02', '\x03' + # Patterns for "smart" cases. if self.smart: self.boundary = re.compile( fr'''(?x) - (?P(?(?{xstart}{etoken}{{1,}}(?![\s{etoken}])(?!$)) + (?P + (? + (? + {xstart}{etoken}{{1,}}(?![\s{etoken}{PUNCT}])(?!$)| + (?:(?<=[\s{etx}{PUNCT}])|^)(? str: else: self.boundary = re.compile( fr'''(?x) - (?P(?(?{etoken}{{1,}}(?![\s{etoken}])(?!$)) + (?P + (? + (? + {etoken}{{1,}}(?![\s{etoken}{PUNCT}])(?!$)| + (?:(?<=[\s{etx}{PUNCT}])|^)(? + a*"foo"* - + *€*charlie. - + test * a * foo*bar* @@ -456,7 +456,7 @@ def test_commonmark(self): *foo bar * - + *(*foo) *(*foo*)* @@ -480,7 +480,7 @@ def test_commonmark(self): ** foo bar** - + a**"foo"** foo**bar** @@ -505,7 +505,7 @@ def test_commonmark(self): **foo bar ** - + **(**foo) *(**foo**)* @@ -696,15 +696,11 @@ def test_commonmark(self): """

foo bar

a * foo bar*

- - - - - +

a*"foo"*

+

*$*alpha.

+

*£*bravo.

+

*€*charlie.

+

test * a *

foobar

5678

@@ -720,9 +716,8 @@ def test_commonmark(self):

*foo bar *

*foo bar *

- - -

(foo)

+

*(*foo)

+

(foo)

foobar

_foo bar _

_(_foo)

@@ -733,8 +728,7 @@ def test_commonmark(self):

(bar).

foo bar

** foo bar**

- - +

a**"foo"**

foobar

foo bar

__ foo bar__

@@ -747,8 +741,7 @@ def test_commonmark(self):

foo, bar, baz

foo-(bar)

**foo bar **

- - +

**(**foo)

(foo)

Gomphocarpus (Gomphocarpus physocarpus, syn. Asclepias physocarpa)

From dbda9ff25de2161fa0115184ff6ec66542d709a4 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 15 Sep 2026 13:16:02 -0600 Subject: [PATCH 28/34] Fix typo --- tests/test_syntax/inline/test_emphasis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_syntax/inline/test_emphasis.py b/tests/test_syntax/inline/test_emphasis.py index 1e015f48a..c4dbd2948 100644 --- a/tests/test_syntax/inline/test_emphasis.py +++ b/tests/test_syntax/inline/test_emphasis.py @@ -426,7 +426,7 @@ def test_commonmark(self): *€*charlie. - + test * a * foo*bar* @@ -700,7 +700,7 @@ def test_commonmark(self):

*$*alpha.

*£*bravo.

*€*charlie.

- +

test * a *

foobar

5678

From f37bef37fb0cd9235ddeeca47838096b7137850e Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 15 Sep 2026 18:24:59 -0600 Subject: [PATCH 29/34] Don't add a zero length delimiter to the stack --- markdown/inlinepatterns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 1a5f4342e..85d662cf5 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -1077,7 +1077,7 @@ def handleMatch( # type: ignore[override] current -= size last -= size de -= size - if not delim.double or last > 1: + if last and (not delim.double or last > 1): stack.append((ds, de, False, last)) # Find opening tokens From c66cda25d5bc9106d27f05753db3d55784790f55 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 15 Sep 2026 18:51:39 -0600 Subject: [PATCH 30/34] Remove dead code As can be seen, the conditional does nothing and has been removed --- markdown/inlinepatterns.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 85d662cf5..961ae76cd 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -1063,22 +1063,19 @@ def handleMatch( # type: ignore[override] continue break - ignore = False - # Create new region if end is valid. - # If not valid, ignore the end but continue parsing. - if not ignore: - is_start = False - ds, de = delimiter[:2] - while current and (not delim.double or current != 1): - size = min(current, 1 if delim.tag_count == 2 and current == 3 else delim.max_size) - new = last - size - self.add_region(delim, ds + new, de, start, start + size, size) - start += size - current -= size - last -= size - de -= size - if last and (not delim.double or last > 1): - stack.append((ds, de, False, last)) + # Create new regions + is_start = False + ds, de = delimiter[:2] + while current and (not delim.double or current != 1): + size = min(current, 1 if delim.tag_count == 2 and current == 3 else delim.max_size) + new = last - size + self.add_region(delim, ds + new, de, start, start + size, size) + start += size + current -= size + last -= size + de -= size + if last and (not delim.double or last > 1): + stack.append((ds, de, False, last)) # Find opening tokens # Looking for: From c9678ebbf8d3ef1da59f6825d867e39b23f07180 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 22 Sep 2026 18:04:57 -0600 Subject: [PATCH 31/34] Minor performance improvement If a delimiter is neither left, right, or ambiguous, greedily capture adjacent delimiters and skip them. --- markdown/inlinepatterns.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 961ae76cd..e42b297ac 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -711,6 +711,8 @@ def _build_patterns(self, token: str) -> str: flags=re.UNICODE ) + self.bad = re.compile(fr'{etoken}+') + return fr'{etoken}' def reset(self) -> None: @@ -953,10 +955,11 @@ def handleMatch( # type: ignore[override] # If token is not an opening, quit m2 = self.get_match(data, m.start(0)) if m2 is None or m2.lastgroup[0] == 'e': # type: ignore[index] - if m2 is not None: - m = m2 + if m2 is None: + delim = self.delimiters[m.group(0)] + m2 = delim.bad.match(data, m.start(0)) # Advance past the full length of the delimiter found - return None, m.start(0), m.end(0) + return None, m2.start(0), m2.end(0) # Get the stack and regions token = data[m.start(0)] From 89ea17817ce0ecd59ad3794a7e210e5a6c380406 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 13 Oct 2026 09:07:50 -0600 Subject: [PATCH 32/34] Move delimiter specific options to the delimiter object --- markdown/extensions/legacy_em.py | 6 +++--- markdown/inlinepatterns.py | 35 ++++++++++++++++---------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index 4ce47176e..0d9cb3857 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -14,7 +14,7 @@ from __future__ import annotations from . import Extension -from ..inlinepatterns import DelimiterProcessor +from ..inlinepatterns import DelimiterProcessor, Delimiter class LegacyEmExtension(Extension): @@ -31,9 +31,9 @@ def extendMarkdown(self, md): # flake8: noqa: E501 27-29 if md.delimiters is not None: - md.delimiters.add('_', 'strong,em') + md.delimiters.add(Delimiter('_', 'strong,em')) else: - md.inlinePatterns.register(DelimiterProcessor('_', 'strong,em', md), 'em_strong', 60) + md.inlinePatterns.register(DelimiterProcessor(Delimiter('_', 'strong,em'), md), 'em_strong', 60) def makeExtension(**kwargs): # pragma: no cover """ Return an instance of the `LegacyEmExtension` """ diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index e42b297ac..6e9cb2439 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -90,8 +90,10 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100) inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) - inlinePatterns.register(DelimiterProcessor('*', 'strong,em', md), 'em_strong', 60) - md.delimiters.add('_', 'strong,em', smart=True) + delim1 = Delimiter('*', 'strong,em') + delim2 = Delimiter('_', 'strong,em', smart=True) + inlinePatterns.register(DelimiterProcessor(delim1, md), 'em_strong', 60) + md.delimiters.add(delim2) return inlinePatterns @@ -643,9 +645,16 @@ def _unescape(m: re.Match[str]) -> str: class Delimiter: """Delimiter.""" - def __init__(self, token: str, tags: str, smart: bool, double: bool): + def __init__( + self, + token: str, + tags: str, + smart: bool = False, + double: bool = False + ): """Initialize.""" + self.token = token self.stack: deque[tuple[int, int, bool, int]] = deque() temp = tags.split(',') self.tag_count = len(temp) @@ -729,11 +738,8 @@ class DelimiterProcessor(InlineProcessor): def __init__( self, - token: str, - tags: str, - md: Markdown, - smart: bool = False, - double: bool = False + delimiter: Delimiter, + md: Markdown ) -> None: """Initialize.""" @@ -747,20 +753,15 @@ def __init__( self.md = md # API for Markdown to pass `safe_mode` into instance self.safe_mode = False - self.add(token, tags, smart, double) + self.add(delimiter) - def add( - self, - token: str, - tags: str, - smart: bool = False, - double: bool = False - ) -> None: + def add(self, delimiter: Delimiter) -> None: """Add a delimiter.""" + token = delimiter.token if token not in self.tokens: self.tokens.append(token) - self.delimiters[token] = Delimiter(token, tags, smart, double) + self.delimiters[token] = delimiter self.pattern = '|'.join([re.escape(t) for t in self.tokens]) self.compiled_re = re.compile(self.pattern, re.DOTALL | re.UNICODE) From 89573fdcdb840e31d17722ef84c6f8815e03f148 Mon Sep 17 00:00:00 2001 From: facelessuser Date: Tue, 13 Oct 2026 10:14:52 -0600 Subject: [PATCH 33/34] Revert "Move delimiter specific options to the delimiter object" MkDocs Autorefs breaks when we do this. It shouldn't, but it isn't worth fighting right now. This reverts commit 89ea17817ce0ecd59ad3794a7e210e5a6c380406. --- markdown/extensions/legacy_em.py | 6 +++--- markdown/inlinepatterns.py | 35 ++++++++++++++++---------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/markdown/extensions/legacy_em.py b/markdown/extensions/legacy_em.py index 0d9cb3857..4ce47176e 100644 --- a/markdown/extensions/legacy_em.py +++ b/markdown/extensions/legacy_em.py @@ -14,7 +14,7 @@ from __future__ import annotations from . import Extension -from ..inlinepatterns import DelimiterProcessor, Delimiter +from ..inlinepatterns import DelimiterProcessor class LegacyEmExtension(Extension): @@ -31,9 +31,9 @@ def extendMarkdown(self, md): # flake8: noqa: E501 27-29 if md.delimiters is not None: - md.delimiters.add(Delimiter('_', 'strong,em')) + md.delimiters.add('_', 'strong,em') else: - md.inlinePatterns.register(DelimiterProcessor(Delimiter('_', 'strong,em'), md), 'em_strong', 60) + md.inlinePatterns.register(DelimiterProcessor('_', 'strong,em', md), 'em_strong', 60) def makeExtension(**kwargs): # pragma: no cover """ Return an instance of the `LegacyEmExtension` """ diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py index 6e9cb2439..e42b297ac 100644 --- a/markdown/inlinepatterns.py +++ b/markdown/inlinepatterns.py @@ -90,10 +90,8 @@ def build_inlinepatterns(md: Markdown, **kwargs: Any) -> util.Registry[InlinePro inlinePatterns.register(SubstituteTagInlineProcessor(LINE_BREAK_RE, 'br'), 'linebreak', 100) inlinePatterns.register(HtmlInlineProcessor(HTML_RE, md), 'html', 90) inlinePatterns.register(HtmlInlineProcessor(ENTITY_RE, md), 'entity', 80) - delim1 = Delimiter('*', 'strong,em') - delim2 = Delimiter('_', 'strong,em', smart=True) - inlinePatterns.register(DelimiterProcessor(delim1, md), 'em_strong', 60) - md.delimiters.add(delim2) + inlinePatterns.register(DelimiterProcessor('*', 'strong,em', md), 'em_strong', 60) + md.delimiters.add('_', 'strong,em', smart=True) return inlinePatterns @@ -645,16 +643,9 @@ def _unescape(m: re.Match[str]) -> str: class Delimiter: """Delimiter.""" - def __init__( - self, - token: str, - tags: str, - smart: bool = False, - double: bool = False - ): + def __init__(self, token: str, tags: str, smart: bool, double: bool): """Initialize.""" - self.token = token self.stack: deque[tuple[int, int, bool, int]] = deque() temp = tags.split(',') self.tag_count = len(temp) @@ -738,8 +729,11 @@ class DelimiterProcessor(InlineProcessor): def __init__( self, - delimiter: Delimiter, - md: Markdown + token: str, + tags: str, + md: Markdown, + smart: bool = False, + double: bool = False ) -> None: """Initialize.""" @@ -753,15 +747,20 @@ def __init__( self.md = md # API for Markdown to pass `safe_mode` into instance self.safe_mode = False - self.add(delimiter) + self.add(token, tags, smart, double) - def add(self, delimiter: Delimiter) -> None: + def add( + self, + token: str, + tags: str, + smart: bool = False, + double: bool = False + ) -> None: """Add a delimiter.""" - token = delimiter.token if token not in self.tokens: self.tokens.append(token) - self.delimiters[token] = delimiter + self.delimiters[token] = Delimiter(token, tags, smart, double) self.pattern = '|'.join([re.escape(t) for t in self.tokens]) self.compiled_re = re.compile(self.pattern, re.DOTALL | re.UNICODE) From 0d989f04f7f80c3e89e5ce9f8ba3c81d116bbb91 Mon Sep 17 00:00:00 2001 From: Waylan Limberg Date: Tue, 13 Oct 2026 14:09:54 -0400 Subject: [PATCH 34/34] Fix typo --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 618bce09a..4798ff7e4 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -20,7 +20,7 @@ See the [Contributing Guide](contributing.md) for details. same paragraph, instead of splitting the paragraph around it (#1643). * Fix issue where inline HTML attributes were rejected if they had `<` or `>` in the attribute (#1647). * Fix issue where an unterminated end tag (`