⬆️ Allow docutils 0.22 (#1084)

This PR updates the project to support docutils 0.22 by adapting to
breaking changes in how docutils serializes boolean attributes and line
numbers. The changes ensure test compatibility across docutils versions
0.20-0.22.

**Changes:**
- Updated docutils version constraint from `<0.22` to `<0.23`
- Introduced `normalize_doctree_xml()` function to handle cross-version
XML serialization differences
- Updated test fixtures to reflect docutils 0.22's line numbering
changes

Note, one issue was found and raised upstream:
sphinx-doc/sphinx#14261

Author: chrisjsewell

.github/workflows/tests.yml

@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        docutils-version: ["0.20", "0.21"]
+        docutils-version: ["0.20", "0.21", "0.22"]
 
     steps:
     - name: Checkout source

AGENTS.md

@@ -85,7 +85,8 @@ tox -e py311-sphinx8
 # Run with coverage
 tox -- --cov=myst_parser
 
-# Update regression test fixtures
+# Update regression test fixtures (this will initially produce an error code if the files change)
+# but note, these files must pass for all python/sphinx/docutils versions
 tox -- --regen-file-failure --force-regen
 ```
 
@@ -337,5 +338,6 @@ flowchart TB
 - [markdown-it-py Documentation](https://markdown-it-py.readthedocs.io/)
 - [Docutils Repository](https://github.com/live-clones/docutils)
 - [Docutils Documentation](https://docutils.sourceforge.io/)
+- [Docutils release log](https://docutils.sourceforge.io/RELEASE-NOTES.html)
 - [Sphinx Repository](https://github.com/sphinx-doc/sphinx)
 - [Sphinx Extension Development](https://www.sphinx-doc.org/en/master/extdev/index.html)

myst_parser/mocking.py

@@ -454,12 +454,18 @@ def run(self) -> list[nodes.Element]:
             literal_block.line = 1  # TODO don;t think this should be 1?
             self.add_name(literal_block)
             if "number-lines" in self.options:
-                try:
-                    startline = int(self.options["number-lines"] or 1)
-                except ValueError as err:
-                    raise DirectiveError(
-                        3, ":number-lines: with non-integer start value"
-                    ) from err
+                # note starting in docutils 0.22 this option is now an integer instead of a string, see: https://github.com/live-clones/docutils/commit/f39ac1413e56a330c8fea6e0d080fed0ff2b8483
+                if self.options["number-lines"] is None:
+                    startline = 1
+                elif isinstance(self.options["number-lines"], int):
+                    startline = self.options["number-lines"]
+                else:
+                    try:
+                        startline = int(self.options["number-lines"] or 1)
+                    except ValueError as err:
+                        raise DirectiveError(
+                            3, ":number-lines: with non-integer start value"
+                        ) from err
                 endline = startline + len(file_content.splitlines())
                 file_content = file_content.removesuffix("\n")
                 tokens = NumberLines([([], file_content)], startline, endline)

pyproject.toml

@@ -34,7 +34,7 @@ keywords = [
 ]
 requires-python = ">=3.11"
 dependencies = [
-    "docutils>=0.20,<0.22",
+    "docutils>=0.20,<0.23",
     "jinja2", # required for substitutions, but let sphinx choose version
     "markdown-it-py~=4.0",
     "mdit-py-plugins~=0.5",
@@ -146,6 +146,7 @@ disallow_any_generics = false
 [tool.pytest.ini_options]
 filterwarnings = [
     "ignore:.*The default for the setting.*:FutureWarning",
+    "ignore:.*:PendingDeprecationWarning",
 ]
 
 [tool.coverage.run]

tests/conftest.py

@@ -0,0 +1,54 @@
+"""Shared pytest configuration for all tests."""
+
+import re
+
+from docutils import __version_info__ as docutils_version_info
+
+DOCUTILS_0_22_PLUS = docutils_version_info >= (0, 22)
+
+
+def normalize_doctree_xml(text: str) -> str:
+    """Normalize docutils XML output for cross-version compatibility.
+
+    In docutils 0.22+, boolean attributes are serialized as "1"/"0"
+    instead of "True"/"False". This function normalizes to the old format
+    for consistent test fixtures.
+    """
+    if DOCUTILS_0_22_PLUS:
+        # Normalize new format (1/0) to old format (1/0)
+        # Only replace when it's clearly a boolean attribute value
+        # Pattern: attribute="1" or attribute="0"
+        attrs = [
+            "force",
+            "glob",
+            "hidden",
+            "id_link",
+            "includehidden",
+            "inline",
+            "internal",
+            "is_div",
+            "linenos",
+            "multi_line_parameter_list",
+            "multi_line_trailing_comma",
+            "no-contents-entry",
+            "no-index",
+            "no-index-entry",
+            "no-typesetting",
+            "no-wrap",
+            "nocontentsentry",
+            "noindex",
+            "noindexentry",
+            "nowrap",
+            "refexplicit",
+            "refspecific",
+            "refwarn",
+            "sorted",
+            "titlesonly",
+            "toctree",
+            "translatable",
+        ]
+        text = re.sub(rf' ({"|".join(attrs)})="1"', r' \1="True"', text)
+        text = re.sub(rf' ({"|".join(attrs)})="0"', r' \1="False"', text)
+        # numbered is changed in math_block, but not in toctree, so we have to be more precise
+        text = re.sub(r' numbered="1" xml:space', r' numbered="True" xml:space', text)
+    return text

tests/test_html/test_html_to_nodes.py

@@ -6,6 +6,7 @@
 
 from myst_parser.config.main import MdParserConfig
 from myst_parser.mdit_to_docutils.html_to_nodes import html_to_nodes
+from tests.conftest import normalize_doctree_xml
 
 FIXTURE_PATH = Path(__file__).parent
 
@@ -32,4 +33,4 @@ def _run_directive(name: str, first_line: str, content: str, position: int):
 def test_html_to_nodes(file_params, mock_renderer):
     output = nodes.container()
     output += html_to_nodes(file_params.content, line_number=0, renderer=mock_renderer)
-    file_params.assert_expected(output.pformat(), rstrip=True)
+    file_params.assert_expected(normalize_doctree_xml(output.pformat()), rstrip=True)

tests/test_renderers/fixtures/docutil_link_resolution.md

@@ -128,14 +128,15 @@
         Test
     <subtitle ids="other test-1" names="other test">
         Other
-    <system_message backrefs="test-1" level="1" line="3" source="<src>/index.md" type="INFO">
-        <paragraph>
-            Duplicate implicit target name: "test".
     <target refid="test-1">
     <paragraph>
         <reference id_link="True" refid="test-1">
             <inline classes="std std-ref">
                 Other
+
+
+<src>/index.md:3: (INFO/1) Target name overrides implicit target name "test".
+<src>/index.md:3: (INFO/1) Hyperlink target "test-1" is not referenced.
 .
 
 [id-with-spaces] 

tests/test_renderers/fixtures/myst-config.txt

@@ -263,6 +263,12 @@ a
 <document source="<string>">
     <paragraph>
         a
+    <section classes="system-messages">
+        <title>
+            Docutils System Messages
+        <system_message level="2" source="<string>" type="WARNING">
+            <paragraph>
+                The `attrs_image` extension is deprecated, please use `attrs_inline` instead. [myst.deprecated]
 
 <string>:: (WARNING/2) The `attrs_image` extension is deprecated, please use `attrs_inline` instead. [myst.deprecated]
 .

tests/test_renderers/test_error_reporting.py

@@ -20,4 +20,9 @@ def test_basic(file_params):
         parser=Parser(),
         settings_overrides={"warning_stream": report_stream},
     )
-    file_params.assert_expected(report_stream.getvalue(), rstrip=True)
+    text = report_stream.getvalue()
+    # changed in docutils 0.23
+    text = text.replace(
+        "corresponding footnote available", "corresponding footnotes available"
+    )
+    file_params.assert_expected(text, rstrip=True)

tests/test_renderers/test_fixtures_docutils.py

@@ -11,6 +11,8 @@
 from typing import Any
 
 import pytest
+from conftest import normalize_doctree_xml
+from docutils import __version_info__ as docutils_version
 from docutils.core import Publisher, publish_doctree
 from pytest_param_files import ParamTestData
 
@@ -37,7 +39,9 @@ def _apply_transforms(self):
     )
 
     # in docutils 0.18 footnote ids have changed
-    outcome = doctree.pformat().replace('"footnote-reference-1"', '"id1"')
+    outcome = normalize_doctree_xml(doctree.pformat()).replace(
+        '"footnote-reference-1"', '"id1"'
+    )
     outcome = outcome.replace(' language=""', "")
     file_params.assert_expected(outcome, rstrip_lines=True)
 
@@ -48,13 +52,18 @@ def test_link_resolution(file_params: ParamTestData):
     settings = settings_from_cmdline(file_params.description)
     report_stream = StringIO()
     settings["warning_stream"] = report_stream
+    if file_params.title == "explicit>implicit":
+        if docutils_version < (0, 22):
+            # reporting changed in docutils 0.22
+            pytest.skip("different in docutils>=0.22")
+        settings["report_level"] = 0
     doctree = publish_doctree(
         file_params.content,
         source_path="<src>/index.md",
         parser=Parser(),
         settings_overrides=settings,
     )
-    outcome = doctree.pformat()
+    outcome = normalize_doctree_xml(doctree.pformat())
     if report_stream.getvalue().strip():
         outcome += "\n\n" + report_stream.getvalue().strip()
     file_params.assert_expected(outcome, rstrip_lines=True)
@@ -75,7 +84,9 @@ def _apply_transforms(self):
         parser=Parser(),
     )
 
-    file_params.assert_expected(doctree.pformat(), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(doctree.pformat()), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "docutil_directives.md")
@@ -95,7 +106,9 @@ def _apply_transforms(self):
         parser=Parser(),
     )
 
-    file_params.assert_expected(doctree.pformat(), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(doctree.pformat()), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "docutil_syntax_extensions.txt")
@@ -109,7 +122,9 @@ def test_syntax_extensions(file_params: ParamTestData):
         parser=Parser(),
         settings_overrides=settings,
     )
-    file_params.assert_expected(doctree.pformat(), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(doctree.pformat()), rstrip_lines=True
+    )
 
 
 def settings_from_cmdline(cmdline: str | None) -> dict[str, Any]: