From a23bca0500837931f6acac990cadb4467cadbe3d Mon Sep 17 00:00:00 2001 From: wanadzhar913 Date: Thu, 4 Jun 2026 21:46:23 +0800 Subject: [PATCH 1/3] fix: merge conflicts Signed-off-by: wanadzhar913 --- docling_core/transforms/serializer/common.py | 184 +++++++++ docling_core/transforms/serializer/html.py | 3 +- .../transforms/serializer/markdown.py | 3 +- .../2408.09869v3_enriched_p2_p3_p5.gt.html | 2 +- .../doc/2408.09869v3_enriched_split.gt.html | 2 +- .../2408.09869v3_enriched_split_p2.gt.html | 2 +- test/data/doc/inline_and_formatting.gt.html | 12 +- test/data/doc/inline_and_formatting.gt.md | 12 +- test/data/doc/polymers.gt.html | 30 +- test/data/doc/polymers.gt.md | 30 +- test/test_serialization.py | 359 +++++++++++++++++- 11 files changed, 590 insertions(+), 49 deletions(-) diff --git a/docling_core/transforms/serializer/common.py b/docling_core/transforms/serializer/common.py index 59f521cfb..d7ee6adaa 100644 --- a/docling_core/transforms/serializer/common.py +++ b/docling_core/transforms/serializer/common.py @@ -6,6 +6,7 @@ import warnings from abc import abstractmethod from collections.abc import Iterable +from enum import Enum from functools import cached_property from pathlib import Path from typing import Annotated, Any, Optional, Union @@ -36,6 +37,7 @@ Span, ) from docling_core.types.doc import ( + CodeItem, ContentLayer, DescriptionAnnotation, DocItem, @@ -44,6 +46,7 @@ FloatingItem, Formatting, FormItem, + FormulaItem, InlineGroup, KeyValueItem, ListGroup, @@ -66,6 +69,14 @@ _logger = logging.getLogger(__name__) +class InlineBoundary(str, Enum): + """Boundary decision between adjacent inline serialization parts.""" + + JOIN = "join" + SPACE = "space" + UNKNOWN = "unknown" + + class _PageBreakNode(NodeItem): """Page break node.""" @@ -184,6 +195,179 @@ def create_ser_result( ) +def _join_inline_parts(parts: list[SerializationResult]) -> str: + """Join inline serialization parts with context-aware spacing.""" + valid_parts = [part for part in parts if part.text] + joined: list[str] = [] + prev_text = "" + prev_item: Optional[DocItem] = None + + for part in valid_parts: + text = part.text + item = part.spans[0].item if part.spans else None + if ( + prev_text + and _classify_inline_boundary( + prev_text=prev_text, + prev_item=prev_item, + text=text, + item=item, + ) + == InlineBoundary.SPACE + ): + joined.append(" ") + joined.append(text) + + prev_text = text + prev_item = item + + return "".join(joined) + + +def _classify_inline_boundary( + *, + prev_text: str, + prev_item: Optional[DocItem], + text: str, + item: Optional[DocItem], +) -> InlineBoundary: + """Classify the boundary between adjacent inline parts.""" + prev_tail = prev_text[-1] + curr_head = text[0] + + if prev_tail.isspace() or curr_head.isspace(): + return InlineBoundary.JOIN + + if prev_item is None or item is None: + return InlineBoundary.UNKNOWN + + if (provenance_boundary := _classify_provenance_boundary(prev_item=prev_item, item=item)) != InlineBoundary.UNKNOWN: + return provenance_boundary + + if isinstance(prev_item, TextItem) and isinstance(item, TextItem): + if not _is_semantic_inline_atom(prev_item) and not _is_semantic_inline_atom(item): + text_boundary = _classify_text_boundary(prev_item=prev_item, item=item) + else: + text_boundary = InlineBoundary.UNKNOWN + if text_boundary != InlineBoundary.UNKNOWN: + return text_boundary + + prev_raw_text = prev_item.text if isinstance(prev_item, TextItem | CodeItem | FormulaItem) else prev_text + curr_raw_text = item.text if isinstance(item, TextItem | CodeItem | FormulaItem) else text + prev_raw_tail = prev_raw_text[-1] if prev_raw_text else prev_tail + curr_raw_head = curr_raw_text[0] if curr_raw_text else curr_head + + # Keep code, formulas, and linked text visually separated from regular text. + if isinstance(prev_item, TextItem) and _is_semantic_inline_atom(item): + return ( + InlineBoundary.SPACE + if prev_raw_tail.isalnum() or prev_raw_tail in {":", ";", ",", "&"} + else InlineBoundary.UNKNOWN + ) + + if _is_semantic_inline_atom(prev_item) and isinstance(item, TextItem): + return ( + InlineBoundary.SPACE if curr_raw_head.isalnum() or curr_raw_head in {"(", "&"} else InlineBoundary.UNKNOWN + ) + + return InlineBoundary.UNKNOWN + + +def _classify_provenance_boundary(*, prev_item: DocItem, item: DocItem) -> InlineBoundary: + """Classify boundary from explicit provenance/original-document signals.""" + prev_orig = getattr(prev_item, "orig", "") + curr_orig = getattr(item, "orig", "") + + if (prev_orig and prev_orig[-1].isspace()) or (curr_orig and curr_orig[0].isspace()): + return InlineBoundary.SPACE + + if prev_item.prov and item.prov: + prev_prov = prev_item.prov[-1] + curr_prov = item.prov[0] + if prev_prov.page_no == curr_prov.page_no: + gap = curr_prov.charspan[0] - prev_prov.charspan[1] + if gap == 0: + return InlineBoundary.JOIN + if gap > 0: + return InlineBoundary.SPACE + + return InlineBoundary.UNKNOWN + + +def _classify_text_boundary(*, prev_item: TextItem, item: TextItem) -> InlineBoundary: + """Classify boundary between adjacent text items.""" + prev_raw_text = prev_item.text + curr_raw_text = item.text + prev_raw_tail = prev_raw_text[-1] + curr_raw_head = curr_raw_text[0] + + prev_is_styled = _is_styled_text(prev_item) + curr_is_styled = _is_styled_text(item) + + if curr_raw_text == "&" and prev_raw_tail.isalnum() and prev_is_styled: + return InlineBoundary.SPACE + if prev_raw_text == "&" and curr_raw_head.isalnum() and curr_is_styled: + return InlineBoundary.SPACE + if prev_raw_tail in {":", ";", ","} and curr_is_styled: + return InlineBoundary.SPACE + + if not (prev_raw_tail.isalnum() and curr_raw_head.isalnum()): + return InlineBoundary.UNKNOWN + + if (len(prev_raw_text) == 1 and prev_is_styled and not curr_is_styled) or ( + len(curr_raw_text) == 1 and curr_is_styled and not prev_is_styled + ): + return InlineBoundary.JOIN + + if prev_is_styled and any(ch.isspace() for ch in prev_raw_text): + return InlineBoundary.SPACE + if curr_is_styled and any(ch.isspace() for ch in curr_raw_text): + return InlineBoundary.SPACE + + if prev_is_styled and curr_is_styled: + return InlineBoundary.SPACE + + if prev_is_styled != curr_is_styled: + return _classify_ambiguous_word_boundary(curr_raw_text=curr_raw_text) + + return InlineBoundary.UNKNOWN + + +def _is_styled_text(item: TextItem) -> bool: + """Return whether a TextItem carries visible inline styling or hyperlink.""" + formatting = item.formatting + has_non_default_formatting = bool( + formatting + and ( + formatting.bold + or formatting.italic + or formatting.underline + or formatting.strikethrough + or formatting.script != Script.BASELINE + ) + ) + return has_non_default_formatting or bool(item.hyperlink) + + +def _is_semantic_inline_atom(item: Optional[DocItem]) -> bool: + """Return whether an inline item should be visually separated from regular text.""" + if isinstance(item, CodeItem | FormulaItem): + return True + return isinstance(item, TextItem) and bool(item.hyperlink) + + +def _classify_ambiguous_word_boundary(*, curr_raw_text: str) -> InlineBoundary: + """Fallback for synthetic boundaries without source position data. + + Without source whitespace or contiguous char spans, this boundary is inherently + ambiguous. Prefer joining short lowercase continuations (e.g. ``Pars`` + ``ing``) + and readability otherwise. + """ + if curr_raw_text.isalpha() and curr_raw_text.islower() and len(curr_raw_text) <= 3: + return InlineBoundary.JOIN + return InlineBoundary.SPACE + + class CommonParams(BaseModel): """Common serialization parameters.""" diff --git a/docling_core/transforms/serializer/html.py b/docling_core/transforms/serializer/html.py index 862ee184a..afbcd0fb4 100644 --- a/docling_core/transforms/serializer/html.py +++ b/docling_core/transforms/serializer/html.py @@ -35,6 +35,7 @@ CommonParams, DocSerializer, _get_annotation_text, + _join_inline_parts, _should_use_legacy_annotations, create_ser_result, ) @@ -892,7 +893,7 @@ def serialize( ) # Join all parts without separators - inline_html = " ".join([p.text for p in parts if p.text]) + inline_html = _join_inline_parts(parts) # Wrap in span if needed if inline_html: diff --git a/docling_core/transforms/serializer/markdown.py b/docling_core/transforms/serializer/markdown.py index 066788d23..fd01609f1 100644 --- a/docling_core/transforms/serializer/markdown.py +++ b/docling_core/transforms/serializer/markdown.py @@ -30,6 +30,7 @@ CommonParams, DocSerializer, _get_annotation_text, + _join_inline_parts, _should_use_legacy_annotations, create_ser_result, ) @@ -806,7 +807,7 @@ def serialize( visited=my_visited, **kwargs, ) - text_res = " ".join([p.text for p in parts if p.text]) + text_res = _join_inline_parts(parts) return create_ser_result(text=text_res, span_source=parts) diff --git a/test/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.html b/test/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.html index 160f63017..1effb65c8 100644 --- a/test/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.html +++ b/test/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.html @@ -154,7 +154,7 @@
  • Can leverage different accelerators (GPU, MPS, etc).

    • 2 Getting Started

    -To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at github.com/DS4SD/docling . All required model assets 1 are downloaded to a local huggingface datasets cache on first use, unless you choose to pre-install the model assets in advance. +To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at github.com/DS4SD/docling. All required model assets 1 are downloaded to a local huggingface datasets cache on first use, unless you choose to pre-install the model assets in advance.

    Docling provides an easy code interface to convert PDF documents from file system, URLs or binary streams, and retrieve the output in either JSON or Markdown format. For convenience, separate methods are offered to convert single documents or batches of documents. A basic usage example is illustrated below. Further examples are available in the Doclign code repository.

    from docling.document_converter import DocumentConverter Large
    source = "https://arxiv.org/pdf/2206.01062" # PDF path or URL converter = DocumentConverter() result = converter.convert_single(source) print(result.render_as_markdown()) # output: "## DocLayNet: A Human -Annotated Dataset for Document -Layout Analysis [...]"
    diff --git a/test/data/doc/2408.09869v3_enriched_split.gt.html b/test/data/doc/2408.09869v3_enriched_split.gt.html index 9f58bd6e9..bd2540bea 100644 --- a/test/data/doc/2408.09869v3_enriched_split.gt.html +++ b/test/data/doc/2408.09869v3_enriched_split.gt.html @@ -175,7 +175,7 @@

    1 Introduction

  • Can leverage different accelerators (GPU, MPS, etc).

    • 2 Getting Started

    -To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at github.com/DS4SD/docling . All required model assets 1 are downloaded to a local huggingface datasets cache on first use, unless you choose to pre-install the model assets in advance. +To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at github.com/DS4SD/docling. All required model assets 1 are downloaded to a local huggingface datasets cache on first use, unless you choose to pre-install the model assets in advance.

    Docling provides an easy code interface to convert PDF documents from file system, URLs or binary streams, and retrieve the output in either JSON or Markdown format. For convenience, separate methods are offered to convert single documents or batches of documents. A basic usage example is illustrated below. Further examples are available in the Doclign code repository.

    from docling.document_converter import DocumentConverter Large
    source = "https://arxiv.org/pdf/2206.01062" # PDF path or URL converter = DocumentConverter() result = converter.convert_single(source) print(result.render_as_markdown()) # output: "## DocLayNet: A Human -Annotated Dataset for Document -Layout Analysis [...]"
    diff --git a/test/data/doc/2408.09869v3_enriched_split_p2.gt.html b/test/data/doc/2408.09869v3_enriched_split_p2.gt.html index 8ae65bc51..e306b22c6 100644 --- a/test/data/doc/2408.09869v3_enriched_split_p2.gt.html +++ b/test/data/doc/2408.09869v3_enriched_split_p2.gt.html @@ -155,7 +155,7 @@
  • Can leverage different accelerators (GPU, MPS, etc).

    • 2 Getting Started

    -To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at github.com/DS4SD/docling . All required model assets 1 are downloaded to a local huggingface datasets cache on first use, unless you choose to pre-install the model assets in advance. +To use Docling, you can simply install the docling package from PyPI. Documentation and examples are available in our GitHub repository at github.com/DS4SD/docling. All required model assets 1 are downloaded to a local huggingface datasets cache on first use, unless you choose to pre-install the model assets in advance.

    Docling provides an easy code interface to convert PDF documents from file system, URLs or binary streams, and retrieve the output in either JSON or Markdown format. For convenience, separate methods are offered to convert single documents or batches of documents. A basic usage example is illustrated below. Further examples are available in the Doclign code repository.

    from docling.document_converter import DocumentConverter Large
    source = "https://arxiv.org/pdf/2206.01062" # PDF path or URL converter = DocumentConverter() result = converter.convert_single(source) print(result.render_as_markdown()) # output: "## DocLayNet: A Human -Annotated Dataset for Document -Layout Analysis [...]"
    diff --git a/test/data/doc/inline_and_formatting.gt.html b/test/data/doc/inline_and_formatting.gt.html index b6150e156..a81646ae0 100644 --- a/test/data/doc/inline_and_formatting.gt.html +++ b/test/data/doc/inline_and_formatting.gt.html @@ -175,20 +175,20 @@

    Contribution guideline example

    This is simple.

    -Foo emphasis strong emphasis both . -Create your feature branch: git checkout -b feature/AmazingFeature . +Foo emphasis strong emphasis both. +Create your feature branch: git checkout -b feature/AmazingFeature.
    1. -Pull the repository . +Pull the repository.
    2. -Create your feature branch ( git checkout -b feature/AmazingFeature ) +Create your feature branch (git checkout -b feature/AmazingFeature)
    3. -Commit your changes ( git commit -m 'Add some AmazingFeature' ) +Commit your changes (git commit -m 'Add some AmazingFeature')
    4. -Push to the branch ( git push origin feature/AmazingFeature ) +Push to the branch (git push origin feature/AmazingFeature)
    5. Open a Pull Request
    6. Whole list item has same formatting
      7. diff --git a/test/data/doc/inline_and_formatting.gt.md b/test/data/doc/inline_and_formatting.gt.md index 0455b064f..3208628db 100644 --- a/test/data/doc/inline_and_formatting.gt.md +++ b/test/data/doc/inline_and_formatting.gt.md @@ -2,14 +2,14 @@ This is simple. -Foo *emphasis* **strong emphasis** ***both*** . +Foo *emphasis* **strong emphasis** ***both***. -Create your feature branch: `git checkout -b feature/AmazingFeature` . +Create your feature branch: `git checkout -b feature/AmazingFeature`. -1. Pull the [**repository**](https://github.com/docling-project/docling) . -2. Create your feature branch ( `git checkout -b feature/AmazingFeature` ) -3. Commit your changes ( `git commit -m 'Add some AmazingFeature'` ) -4. Push to the branch ( `git push origin feature/AmazingFeature` ) +1. Pull the [**repository**](https://github.com/docling-project/docling). +2. Create your feature branch (`git checkout -b feature/AmazingFeature`) +3. Commit your changes (`git commit -m 'Add some AmazingFeature'`) +4. Push to the branch (`git push origin feature/AmazingFeature`) 5. Open a Pull Request 6. **Whole list item has same formatting** 7. List item has *mixed or partial* formatting diff --git a/test/data/doc/polymers.gt.html b/test/data/doc/polymers.gt.html index 32ac45341..a5c64456a 100644 --- a/test/data/doc/polymers.gt.html +++ b/test/data/doc/polymers.gt.html @@ -237,10 +237,10 @@

      Safety and Regulatory Considerations

      Extraction in food simulants
      • -What it is : Samples of the packaging material are immersed in a liquid that mimics the chemical properties of a specific food type (e.g., aqueous, acidic, fatty). +What it is: Samples of the packaging material are immersed in a liquid that mimics the chemical properties of a specific food type (e.g., aqueous, acidic, fatty).
      • -Typical simulants : +Typical simulants:
        • 3% acetic acid (for acidic foods)
        • 50% ethanol (for alcohol‑based foods)
          • @@ -249,7 +249,7 @@

          Safety and Regulatory Considerations

      • -Procedure : +Procedure:
        • Prepare a defined volume of simulant in a sealed vessel.
        • Immerse the material for a set time at a controlled temperature (often 50 °C–70 °C).
          • @@ -257,13 +257,13 @@

          Safety and Regulatory Considerations

      • -Analysis : GC‑MS, LC‑MS, or HPLC depending on the analyte class. +Analysis: GC‑MS, LC‑MS, or HPLC depending on the analyte class.
      • -Advantages : Direct assessment of potential migration into a realistic medium; scalable for routine testing. +Advantages: Direct assessment of potential migration into a realistic medium; scalable for routine testing.
      • -Limitations : Does not account for headspace gas migration; may underestimate migration of highly volatile substances. +Limitations: Does not account for headspace gas migration; may underestimate migration of highly volatile substances.
      @@ -271,10 +271,10 @@

      Safety and Regulatory Considerations

      Headspace analysis
      • -What it is : Measurement of volatile substances that migrate from the material into the surrounding gas phase. +What it is: Measurement of volatile substances that migrate from the material into the surrounding gas phase.
      • -Procedure : +Procedure:
        • Seal the material in a headspace vial or chamber.
        • Equilibrate at a defined temperature (commonly 25 °C–60 °C).
          • @@ -283,13 +283,13 @@

          Safety and Regulatory Considerations

      • -Applications : Assessment of aromas, flavor compounds, or volatile contaminants. +Applications: Assessment of aromas, flavor compounds, or volatile contaminants.
      • -Advantages : Sensitive to low‑concentration volatiles; minimal sample preparation. +Advantages: Sensitive to low‑concentration volatiles; minimal sample preparation.
      • -Limitations : Does not capture non‑volatile migration; results depend on equilibrium time and temperature. +Limitations: Does not capture non‑volatile migration; results depend on equilibrium time and temperature.
      @@ -297,10 +297,10 @@

      Safety and Regulatory Considerations

      Direct contact tests
      • -What it is : The packaging material is placed in direct contact with the food or food simulant, often using a defined food‑packaging configuration. +What it is: The packaging material is placed in direct contact with the food or food simulant, often using a defined food‑packaging configuration.
      • -Procedure : +Procedure:
        • Assemble the material and food (or simulant) in a mold or container that simulates real usage (e.g., sealed pouch, jar).
        • Incubate for the intended storage time at the relevant temperature.
          • @@ -309,10 +309,10 @@

          Safety and Regulatory Considerations

      • -Advantages : Mimics real consumer exposure; captures both liquid and vapor migration pathways. +Advantages: Mimics real consumer exposure; captures both liquid and vapor migration pathways.
      • -Limitations : More labor‑intensive; requires careful control of contact area, thickness, and sealing integrity. +Limitations: More labor‑intensive; requires careful control of contact area, thickness, and sealing integrity.
      diff --git a/test/data/doc/polymers.gt.md b/test/data/doc/polymers.gt.md index e44bcb8d1..3bc75356c 100644 --- a/test/data/doc/polymers.gt.md +++ b/test/data/doc/polymers.gt.md @@ -50,38 +50,38 @@ **Common migration testing methods** - **Extraction in food simulants** - - *What it is* : Samples of the packaging material are immersed in a liquid that mimics the chemical properties of a specific food type (e.g., aqueous, acidic, fatty). - - *Typical simulants* : + - *What it is*: Samples of the packaging material are immersed in a liquid that mimics the chemical properties of a specific food type (e.g., aqueous, acidic, fatty). + - *Typical simulants*: - 3% acetic acid (for acidic foods) - 50% ethanol (for alcohol‑based foods) - 95% ethanol (for high‑fat foods) - Distilled water (for aqueous foods) - - *Procedure* : + - *Procedure*: - Prepare a defined volume of simulant in a sealed vessel. - Immerse the material for a set time at a controlled temperature (often 50 °C–70 °C). - Remove, filter, and concentrate the extract for analysis. - - *Analysis* : GC‑MS, LC‑MS, or HPLC depending on the analyte class. - - *Advantages* : Direct assessment of potential migration into a realistic medium; scalable for routine testing. - - *Limitations* : Does not account for headspace gas migration; may underestimate migration of highly volatile substances. + - *Analysis*: GC‑MS, LC‑MS, or HPLC depending on the analyte class. + - *Advantages*: Direct assessment of potential migration into a realistic medium; scalable for routine testing. + - *Limitations*: Does not account for headspace gas migration; may underestimate migration of highly volatile substances. - **Headspace analysis** - - *What it is* : Measurement of volatile substances that migrate from the material into the surrounding gas phase. - - *Procedure* : + - *What it is*: Measurement of volatile substances that migrate from the material into the surrounding gas phase. + - *Procedure*: - Seal the material in a headspace vial or chamber. - Equilibrate at a defined temperature (commonly 25 °C–60 °C). - Sample the gas phase with a gas sampling needle or syringe. - Analyze via GC‑FID, GC‑MS, or PTR‑MS. - - *Applications* : Assessment of aromas, flavor compounds, or volatile contaminants. - - *Advantages* : Sensitive to low‑concentration volatiles; minimal sample preparation. - - *Limitations* : Does not capture non‑volatile migration; results depend on equilibrium time and temperature. + - *Applications*: Assessment of aromas, flavor compounds, or volatile contaminants. + - *Advantages*: Sensitive to low‑concentration volatiles; minimal sample preparation. + - *Limitations*: Does not capture non‑volatile migration; results depend on equilibrium time and temperature. - **Direct contact tests** - - *What it is* : The packaging material is placed in direct contact with the food or food simulant, often using a defined food‑packaging configuration. - - *Procedure* : + - *What it is*: The packaging material is placed in direct contact with the food or food simulant, often using a defined food‑packaging configuration. + - *Procedure*: - Assemble the material and food (or simulant) in a mold or container that simulates real usage (e.g., sealed pouch, jar). - Incubate for the intended storage time at the relevant temperature. - Extract or sample the food directly (e.g., through the material or by taking a portion of the food). - Analyze for migrated substances. - - *Advantages* : Mimics real consumer exposure; captures both liquid and vapor migration pathways. - - *Limitations* : More labor‑intensive; requires careful control of contact area, thickness, and sealing integrity. + - *Advantages*: Mimics real consumer exposure; captures both liquid and vapor migration pathways. + - *Limitations*: More labor‑intensive; requires careful control of contact area, thickness, and sealing integrity. These three approaches—extraction in food simulants, headspace analysis, and direct contact tests—complement each other to provide a comprehensive assessment of potential migration from packaging into food. diff --git a/test/test_serialization.py b/test/test_serialization.py index b6bd4b1db..dbb336289 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -7,7 +7,18 @@ import pytest -from docling_core.transforms.serializer.common import _DEFAULT_LABELS +from docling_core.transforms.serializer.common import ( + _DEFAULT_LABELS, + InlineBoundary, + _classify_ambiguous_word_boundary, + _classify_inline_boundary, + _classify_provenance_boundary, + _classify_text_boundary, + _is_semantic_inline_atom, + _is_styled_text, + _join_inline_parts, + create_ser_result, +) from docling_core.transforms.serializer.html import ( HTMLDocSerializer, HTMLMetaSerializer, @@ -24,18 +35,22 @@ ) from docling_core.transforms.serializer.webvtt import WebVTTDocSerializer, WebVTTParams from docling_core.transforms.visualizer.layout_visualizer import LayoutVisualizer + from docling_core.types.doc import DoclingDocument -from docling_core.types.doc.base import ImageRefMode +from docling_core.types.doc.base import BoundingBox, ImageRefMode from docling_core.types.doc.document import ( BaseMeta, CharSpan, DescriptionAnnotation, + DoclingDocument, EntitiesMetaField, EntityMention, + Formatting, LanguageMetaField, PictureClassificationMetaField, PictureClassificationPrediction, PictureMeta, + ProvenanceItem, RefItem, RichTableCell, SummaryMetaField, @@ -69,6 +84,36 @@ def _normalize_quotes(s: str) -> str: assert actual == expected +def _make_inline_doc() -> tuple[DoclingDocument, object]: + doc = DoclingDocument(name="test") + return doc, doc.add_inline_group() + + +def _add_inline_text( + doc: DoclingDocument, + group, + text: str, + *, + label: DocItemLabel = DocItemLabel.TEXT, + **kwargs, +): + return doc.add_text(label=label, parent=group, text=text, **kwargs) + + +def _extract_body_content(html: str) -> str: + start = html.find("") + 6 + end = html.find("") + return html[start:end].strip() + + +def _make_prov(start: int, end: int, *, page_no: int = 1) -> ProvenanceItem: + return ProvenanceItem( + page_no=page_no, + bbox=BoundingBox(l=0.0, t=0.0, r=1.0, b=1.0), + charspan=(start, end), + ) + + # =============================== # Markdown tests # =============================== @@ -1103,3 +1148,313 @@ def test_html_meta_emits_xhtml_compatible_attributes(): assert 'data-meta-name="entities"' in html_out # Output must be parseable by a strict XML parser. ET.fromstring(html_out) + + +# =============================== +# Tests for inline group join behavior without spaces +# =============================== + + +def test_md_inline_group_no_spaces(): + """Test that inline groups join text parts without spaces for continuous text.""" + doc, group = _make_inline_doc() + _add_inline_text( + doc, + group, + label=DocItemLabel.TEXT, + text="D", + formatting=Formatting( + bold=True, + italic=False, + underline=False, + strikethrough=False, + script="baseline", + ), + ) + _add_inline_text( + doc, + group, + label=DocItemLabel.TEXT, + text="ocling", + formatting=Formatting( + bold=False, + italic=False, + underline=False, + strikethrough=False, + script="baseline", + ), + ) + + # This should serialize as "**D**ocling" without space + ser = MarkdownDocSerializer(doc=doc) + actual = ser.serialize().text.strip() + + expected = "**D**ocling" + assert actual == expected + + +def test_html_inline_group_no_spaces(): + """Test that inline groups join text parts without spaces for continuous text.""" + doc, group = _make_inline_doc() + _add_inline_text( + doc, + group, + label=DocItemLabel.TEXT, + text="Project", + formatting=Formatting( + bold=True, + italic=False, + underline=False, + strikethrough=False, + script="baseline", + ), + ) + _add_inline_text( + doc, + group, + label=DocItemLabel.TEXT, + text="ing", + formatting=Formatting( + bold=False, + italic=False, + underline=False, + strikethrough=False, + script="baseline", + ), + ) + + # This should serialize as Projecting without space + ser = HTMLDocSerializer( + doc=doc, params=HTMLParams(html_head="", prettify=False) + ) + actual = ser.serialize().text + + body_content = _extract_body_content(actual) + assert "Projecting" in body_content + + +def test_md_inline_group_mixed_formatting_mid_word(): + """Test inline group with different formatting mid-word.""" + doc, group = _make_inline_doc() + _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="Pars") + _add_inline_text( + doc, + group, + label=DocItemLabel.TEXT, + text="ing", + formatting=Formatting( + bold=False, + italic=True, + underline=False, + strikethrough=False, + script="baseline", + ), + ) + + ser = MarkdownDocSerializer(doc=doc) + actual = ser.serialize().text.strip() + + expected = "Pars*ing*" + assert actual == expected + + +def test_html_inline_group_mixed_formatting_mid_word(): + """Test inline group with different formatting mid-word.""" + doc, group = _make_inline_doc() + _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="Pars") + _add_inline_text( + doc, + group, + label=DocItemLabel.TEXT, + text="ing", + formatting=Formatting( + bold=False, + italic=True, + underline=False, + strikethrough=False, + script="baseline", + ), + ) + + ser = HTMLDocSerializer( + doc=doc, params=HTMLParams(html_head="", prettify=False) + ) + actual = ser.serialize().text + + body_content = _extract_body_content(actual) + assert "Parsing" in body_content.replace("\n", " ") + + +def test_md_inline_group_single_part(): + """Test inline group with single text part (no joining needed).""" + doc, group = _make_inline_doc() + _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="Single") + + ser = MarkdownDocSerializer(doc=doc) + actual = ser.serialize().text.strip() + + expected = "Single" + assert actual == expected + + +def test_html_inline_group_single_part(): + """Test inline group with single text part (no joining needed).""" + doc, group = _make_inline_doc() + _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="Single") + + ser = HTMLDocSerializer( + doc=doc, params=HTMLParams(html_head="", prettify=False) + ) + actual = ser.serialize().text + + body_content = _extract_body_content(actual) + assert "Single" in body_content + + +def test_join_inline_parts_filters_empty_parts_and_inserts_spacing(): + doc, group = _make_inline_doc() + text_item = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="A hyperlink on") + code_item = doc.add_code(text="code in a line", parent=group, hyperlink="#link") + + parts = [ + create_ser_result(text="", span_source=text_item), + create_ser_result(text="A hyperlink on", span_source=text_item), + create_ser_result(text="[`code in a line`](#link)", span_source=code_item), + ] + + assert _join_inline_parts(parts) == "A hyperlink on [`code in a line`](#link)" + + +@pytest.mark.parametrize( + ("prev_orig", "curr_orig", "prev_prov", "curr_prov", "expected"), + [ + ("Hello ", "world", None, None, InlineBoundary.SPACE), + ("Hello", "world", _make_prov(0, 5), _make_prov(5, 10), InlineBoundary.JOIN), + ("Hello", "world", _make_prov(0, 5), _make_prov(6, 11), InlineBoundary.SPACE), + ("Hello", "world", _make_prov(0, 5, page_no=1), _make_prov(0, 5, page_no=2), InlineBoundary.UNKNOWN), + ], +) +def test_classify_source_boundary(prev_orig, curr_orig, prev_prov, curr_prov, expected): + doc, group = _make_inline_doc() + prev_item = _add_inline_text( + doc, group, label=DocItemLabel.TEXT, text=prev_orig.strip(), orig=prev_orig, prov=prev_prov + ) + curr_item = _add_inline_text( + doc, group, label=DocItemLabel.TEXT, text=curr_orig.strip(), orig=curr_orig, prov=curr_prov + ) + + assert _classify_provenance_boundary(prev_item=prev_item, item=curr_item) == expected + + +def test_classify_inline_boundary_handles_missing_items_and_whitespace(): + assert ( + _classify_inline_boundary(prev_text="foo ", prev_item=None, text="bar", item=None) + == InlineBoundary.JOIN + ) + assert ( + _classify_inline_boundary(prev_text="foo", prev_item=None, text="bar", item=None) + == InlineBoundary.UNKNOWN + ) + + +@pytest.mark.parametrize( + ("prev_text", "prev_formatting", "prev_hyperlink", "curr_text", "curr_formatting", "curr_hyperlink", "expected"), + [ + ("snippet:", None, None, "bold", Formatting(bold=True), None, InlineBoundary.SPACE), + ("D", Formatting(bold=True), None, "ocling", None, None, InlineBoundary.JOIN), + ("Pars", None, None, "ing", Formatting(italic=True), None, InlineBoundary.JOIN), + ("Foo", None, None, "emphasis", Formatting(italic=True), None, InlineBoundary.SPACE), + ("strong emphasis", Formatting(bold=True), None, "tail", None, None, InlineBoundary.SPACE), + ("bold", Formatting(bold=True), None, "italic", Formatting(italic=True), None, InlineBoundary.SPACE), + ("bold", Formatting(bold=True), None, "&", None, None, InlineBoundary.SPACE), + ("hello-", None, None, "world", Formatting(italic=True), None, InlineBoundary.UNKNOWN), + ], +) +def test_classify_text_boundary( + prev_text, + prev_formatting, + prev_hyperlink, + curr_text, + curr_formatting, + curr_hyperlink, + expected, +): + doc, group = _make_inline_doc() + prev_item = _add_inline_text( + doc, + group, + label=DocItemLabel.TEXT, + text=prev_text, + formatting=prev_formatting, + hyperlink=prev_hyperlink, + ) + curr_item = _add_inline_text( + doc, + group, + label=DocItemLabel.TEXT, + text=curr_text, + formatting=curr_formatting, + hyperlink=curr_hyperlink, + ) + + assert _classify_text_boundary(prev_item=prev_item, item=curr_item) == expected + + +def test_inline_boundary_separates_semantic_atoms_from_text(): + doc, group = _make_inline_doc() + prev_item = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="A hyperlink on") + code_item = doc.add_code(text="code in a line", parent=group, hyperlink="#link") + formula_item = doc.add_formula(text="E=mc^2", parent=group) + next_item = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="(inline)") + + assert ( + _classify_inline_boundary( + prev_text="A hyperlink on", + prev_item=prev_item, + text="[`code in a line`](#link)", + item=code_item, + ) + == InlineBoundary.SPACE + ) + assert ( + _classify_inline_boundary( + prev_text="$E=mc^2$", + prev_item=formula_item, + text="(inline)", + item=next_item, + ) + == InlineBoundary.SPACE + ) + + +def test_common_inline_helper_flags(): + doc, group = _make_inline_doc() + plain = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="plain") + bold = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="bold", formatting=Formatting(bold=True)) + linked = _add_inline_text( + doc, group, label=DocItemLabel.TEXT, text="link", hyperlink="https://example.com" + ) + code = doc.add_code(text="print()", parent=group) + formula = doc.add_formula(text="E=mc^2", parent=group) + + assert _is_styled_text(plain) is False + assert _is_styled_text(bold) is True + assert _is_styled_text(linked) is True + + assert _is_semantic_inline_atom(plain) is False + assert _is_semantic_inline_atom(linked) is True + assert _is_semantic_inline_atom(code) is True + assert _is_semantic_inline_atom(formula) is True + + +@pytest.mark.parametrize( + ("curr_raw_text", "expected"), + [ + ("ing", InlineBoundary.JOIN), + ("emphasis", InlineBoundary.SPACE), + ("XYZ", InlineBoundary.SPACE), + ("42", InlineBoundary.SPACE), + ], +) +def test_classify_ambiguous_word_boundary(curr_raw_text, expected): + assert _classify_ambiguous_word_boundary(curr_raw_text=curr_raw_text) == expected From 6f0ba5dd06ff24a94cf46a5373b86ed9640044ca Mon Sep 17 00:00:00 2001 From: wanadzhar913 Date: Tue, 23 Jun 2026 01:00:08 +0800 Subject: [PATCH 2/3] fix: account for noisy OCR layout extraction/markdown specific boundaries/styled text handling by letting text heuristics override provenance only when it is safe Signed-off-by: wanadzhar913 --- docling_core/transforms/serializer/common.py | 317 ++++++++++- test/test_serialization.py | 539 ++++++++++++++++++- 2 files changed, 828 insertions(+), 28 deletions(-) diff --git a/docling_core/transforms/serializer/common.py b/docling_core/transforms/serializer/common.py index d7ee6adaa..f59b90fc1 100644 --- a/docling_core/transforms/serializer/common.py +++ b/docling_core/transforms/serializer/common.py @@ -77,6 +77,13 @@ class InlineBoundary(str, Enum): UNKNOWN = "unknown" +_RIGHT_ATTACHING_CHARS = frozenset({")", "]", "}", ",", ";", ":", ".", "!", "?", "%"}) +_BRACKET_OPENERS = frozenset({"(", "[", "{"}) +_WORD_JOIN_CHARS = frozenset({"-", "/"}) +_QUOTE_CHARS = frozenset({"'", '"', "\u2018", "\u2019", "\u201c", "\u201d"}) +_MARKDOWN_EMPH_CHARS = ("*", "_") + + class _PageBreakNode(NodeItem): """Page break node.""" @@ -239,18 +246,31 @@ def _classify_inline_boundary( return InlineBoundary.JOIN if prev_item is None or item is None: - return InlineBoundary.UNKNOWN + return _classify_character_boundary(prev_tail=prev_tail, curr_head=curr_head) - if (provenance_boundary := _classify_provenance_boundary(prev_item=prev_item, item=item)) != InlineBoundary.UNKNOWN: - return provenance_boundary + provenance_boundary = _classify_provenance_boundary(prev_item=prev_item, item=item) if isinstance(prev_item, TextItem) and isinstance(item, TextItem): if not _is_semantic_inline_atom(prev_item) and not _is_semantic_inline_atom(item): text_boundary = _classify_text_boundary(prev_item=prev_item, item=item) - else: - text_boundary = InlineBoundary.UNKNOWN - if text_boundary != InlineBoundary.UNKNOWN: - return text_boundary + if _should_text_boundary_override_provenance( + prev_item=prev_item, + item=item, + text_boundary=text_boundary, + provenance_boundary=provenance_boundary, + ): + return text_boundary + + if provenance_boundary != InlineBoundary.UNKNOWN: + if provenance_boundary == InlineBoundary.SPACE: + return InlineBoundary.SPACE + if not _should_provenance_join_override_spacing( + prev_item=prev_item, + item=item, + prev_text=prev_text, + text=text, + ): + return InlineBoundary.SPACE prev_raw_text = prev_item.text if isinstance(prev_item, TextItem | CodeItem | FormulaItem) else prev_text curr_raw_text = item.text if isinstance(item, TextItem | CodeItem | FormulaItem) else text @@ -258,19 +278,33 @@ def _classify_inline_boundary( curr_raw_head = curr_raw_text[0] if curr_raw_text else curr_head # Keep code, formulas, and linked text visually separated from regular text. - if isinstance(prev_item, TextItem) and _is_semantic_inline_atom(item): - return ( - InlineBoundary.SPACE - if prev_raw_tail.isalnum() or prev_raw_tail in {":", ";", ",", "&"} - else InlineBoundary.UNKNOWN + if isinstance(prev_item, TextItem) and not _is_semantic_inline_atom(prev_item) and _is_semantic_inline_atom(item): + if prev_raw_tail.isalnum() or prev_raw_tail in {":", ";", ",", "&"}: + return InlineBoundary.SPACE + return _classify_rendered_character_boundary( + prev_tail=prev_raw_tail, + curr_head=curr_raw_head, + prev_text=prev_text, ) if _is_semantic_inline_atom(prev_item) and isinstance(item, TextItem): - return ( - InlineBoundary.SPACE if curr_raw_head.isalnum() or curr_raw_head in {"(", "&"} else InlineBoundary.UNKNOWN + if ( + curr_raw_head.isalnum() + or curr_raw_head in {"(", "&", "[", "'"} + or (text and text[0] in {"(", "&", "[", "'"}) + ): + return InlineBoundary.SPACE + return _classify_rendered_character_boundary( + prev_tail=prev_text[-1] if prev_text else prev_raw_tail, + curr_head=text[0] if text else curr_raw_head, + prev_text=prev_text, ) - return InlineBoundary.UNKNOWN + return _classify_rendered_character_boundary( + prev_tail=prev_raw_tail, + curr_head=curr_raw_head, + prev_text=prev_text, + ) def _classify_provenance_boundary(*, prev_item: DocItem, item: DocItem) -> InlineBoundary: @@ -304,32 +338,172 @@ def _classify_text_boundary(*, prev_item: TextItem, item: TextItem) -> InlineBou prev_is_styled = _is_styled_text(prev_item) curr_is_styled = _is_styled_text(item) - if curr_raw_text == "&" and prev_raw_tail.isalnum() and prev_is_styled: + if curr_raw_text == "&" and prev_raw_tail.isalnum(): return InlineBoundary.SPACE if prev_raw_text == "&" and curr_raw_head.isalnum() and curr_is_styled: return InlineBoundary.SPACE if prev_raw_tail in {":", ";", ","} and curr_is_styled: return InlineBoundary.SPACE + if prev_is_styled and curr_is_styled: + return InlineBoundary.SPACE + + prev_has_non_baseline_script = _has_non_baseline_script(prev_item) + curr_has_non_baseline_script = _has_non_baseline_script(item) + if not (prev_raw_tail.isalnum() and curr_raw_head.isalnum()): - return InlineBoundary.UNKNOWN + if prev_has_non_baseline_script and curr_raw_head in {"+", "-", "*", "/"}: + return InlineBoundary.SPACE + if prev_is_styled and not curr_is_styled and curr_raw_head.isalnum(): + return InlineBoundary.SPACE + if not prev_is_styled and curr_is_styled and prev_raw_tail.isalnum(): + return InlineBoundary.SPACE + return _classify_character_boundary(prev_tail=prev_raw_tail, curr_head=curr_raw_head) + + if prev_has_non_baseline_script or curr_has_non_baseline_script: + return InlineBoundary.SPACE - if (len(prev_raw_text) == 1 and prev_is_styled and not curr_is_styled) or ( - len(curr_raw_text) == 1 and curr_is_styled and not prev_is_styled - ): - return InlineBoundary.JOIN + if len(prev_raw_text) == 1 and prev_is_styled and not curr_is_styled: + curr_raw_head = curr_raw_text[0] if curr_raw_text else "" + first_token = curr_raw_text.split()[0].lower() + if curr_raw_head.islower() and first_token not in _COMMON_SHORT_WORDS: + return InlineBoundary.JOIN + return InlineBoundary.SPACE if prev_is_styled and any(ch.isspace() for ch in prev_raw_text): return InlineBoundary.SPACE if curr_is_styled and any(ch.isspace() for ch in curr_raw_text): return InlineBoundary.SPACE - if prev_is_styled and curr_is_styled: - return InlineBoundary.SPACE - if prev_is_styled != curr_is_styled: + if curr_is_styled and len(curr_raw_text) == 1: + return InlineBoundary.SPACE return _classify_ambiguous_word_boundary(curr_raw_text=curr_raw_text) + return _classify_character_boundary(prev_tail=prev_raw_tail, curr_head=curr_raw_head) + + +def _is_markdown_syntax_join( + *, + prev_tail: str, + curr_head: str, + prev_text: str = "", + text: str = "", +) -> bool: + """Return whether adjacent rendered parts should join for markdown syntax cleanup.""" + if curr_head in _RIGHT_ATTACHING_CHARS: + return True + + if prev_tail in _WORD_JOIN_CHARS: + return True + + if prev_tail in _BRACKET_OPENERS: + return True + + if curr_head in _WORD_JOIN_CHARS: + return True + + if prev_tail in _QUOTE_CHARS and curr_head in _QUOTE_CHARS: + return True + + if prev_text.endswith(_MARKDOWN_EMPH_CHARS) and curr_head in _RIGHT_ATTACHING_CHARS: + return True + + if text.startswith(_MARKDOWN_EMPH_CHARS) and prev_tail.isalnum(): + return False + + return False + + +def _classify_word_punctuation_boundary(*, prev_tail: str, curr_head: str) -> InlineBoundary: + """Classify common word boundaries after punctuation in source text.""" + if prev_tail in {",", ";", ":"} and curr_head.isalnum(): + return InlineBoundary.SPACE + + if prev_tail == "." and (curr_head.isalnum() or curr_head == "["): + return InlineBoundary.SPACE + + if prev_tail == ")" and curr_head == "[": + return InlineBoundary.SPACE + + if curr_head == "&" and prev_tail.isalnum(): + return InlineBoundary.SPACE + + return InlineBoundary.UNKNOWN + + +def _should_provenance_join_override_spacing( + *, + prev_item: DocItem, + item: DocItem, + prev_text: str, + text: str, +) -> bool: + """Return whether a provenance join should suppress an otherwise expected space.""" + if not isinstance(prev_item, TextItem) or not isinstance(item, TextItem): + return True + + if _is_markdown_syntax_join( + prev_tail=prev_text[-1] if prev_text else "", + curr_head=text[0] if text else "", + prev_text=prev_text, + text=text, + ): + return True + + if _is_ocr_safe_midword_join(prev_item=prev_item, item=item): + return True + + prev_raw_tail = prev_item.text[-1] if prev_item.text else "" + curr_raw_head = item.text[0] if item.text else "" + if _classify_word_punctuation_boundary(prev_tail=prev_raw_tail, curr_head=curr_raw_head) == InlineBoundary.SPACE: + return False + + if prev_text.endswith(_MARKDOWN_EMPH_CHARS) and curr_raw_head in _QUOTE_CHARS: + return False + + return True + + +def _classify_rendered_character_boundary( + *, + prev_tail: str, + curr_head: str, + prev_text: str, +) -> InlineBoundary: + """Character fallback using rendered text for markdown emphasis spacing.""" + if prev_text.endswith(_MARKDOWN_EMPH_CHARS) and curr_head in _QUOTE_CHARS: + return InlineBoundary.SPACE + return _classify_character_boundary(prev_tail=prev_tail, curr_head=curr_head) + + +def _classify_character_boundary(*, prev_tail: str, curr_head: str) -> InlineBoundary: + """Classify boundary from adjacent serialized characters when item metadata is inconclusive.""" + if not prev_tail or not curr_head: + return InlineBoundary.UNKNOWN + + word_boundary = _classify_word_punctuation_boundary(prev_tail=prev_tail, curr_head=curr_head) + if word_boundary != InlineBoundary.UNKNOWN: + return word_boundary + + if prev_tail.isalnum() and curr_head.isalnum(): + return InlineBoundary.SPACE + + if prev_tail in _WORD_JOIN_CHARS: + return InlineBoundary.JOIN + + if curr_head in _RIGHT_ATTACHING_CHARS: + return InlineBoundary.JOIN + + if curr_head in _WORD_JOIN_CHARS: + return InlineBoundary.JOIN + + if prev_tail in _BRACKET_OPENERS: + return InlineBoundary.JOIN + + if prev_tail in _QUOTE_CHARS and curr_head in _QUOTE_CHARS: + return InlineBoundary.JOIN + return InlineBoundary.UNKNOWN @@ -363,11 +537,104 @@ def _classify_ambiguous_word_boundary(*, curr_raw_text: str) -> InlineBoundary: ambiguous. Prefer joining short lowercase continuations (e.g. ``Pars`` + ``ing``) and readability otherwise. """ - if curr_raw_text.isalpha() and curr_raw_text.islower() and len(curr_raw_text) <= 3: + if _is_likely_word_suffix(curr_raw_text): return InlineBoundary.JOIN return InlineBoundary.SPACE +_COMMON_SHORT_WORDS = frozenset({"a", "an", "and", "at", "in", "is", "it", "on", "or", "the", "to"}) + + +def _is_likely_word_suffix(curr_raw_text: str) -> bool: + """Return whether adjacent text likely continues the previous token mid-word.""" + if not (curr_raw_text.isalpha() and curr_raw_text.islower() and len(curr_raw_text) <= 3): + return False + return curr_raw_text not in _COMMON_SHORT_WORDS + + +def _has_non_baseline_script(item: TextItem) -> bool: + """Return whether a TextItem uses subscript or superscript.""" + formatting = item.formatting + return bool(formatting and formatting.script != Script.BASELINE) + + +def _is_ocr_safe_midword_join(*, prev_item: TextItem, item: TextItem) -> bool: + """Return whether a join should override provenance spacing for OCR-like splits.""" + if _has_non_baseline_script(prev_item) or _has_non_baseline_script(item): + return False + + prev_raw_text = prev_item.text + curr_raw_text = item.text + prev_is_styled = _is_styled_text(prev_item) + curr_is_styled = _is_styled_text(item) + + if len(prev_raw_text) == 1 and prev_is_styled and not curr_is_styled: + curr_raw_head = curr_raw_text[0] if curr_raw_text else "" + first_token = curr_raw_text.split()[0].lower() + return curr_raw_head.islower() and first_token not in _COMMON_SHORT_WORDS + + return _is_likely_word_suffix(curr_raw_text) and not ( + _is_semantic_inline_atom(prev_item) or _is_semantic_inline_atom(item) + ) + + +def _is_false_ocr_word_join(*, prev_item: TextItem, item: TextItem) -> bool: + """Return whether spacing should override provenance joins between separate words.""" + if _has_non_baseline_script(prev_item) or _has_non_baseline_script(item): + return False + + prev_raw_text = prev_item.text + curr_raw_text = item.text + prev_raw_tail = prev_raw_text[-1] if prev_raw_text else "" + curr_raw_head = curr_raw_text[0] if curr_raw_text else "" + prev_is_styled = _is_styled_text(prev_item) + curr_is_styled = _is_styled_text(item) + + if not prev_is_styled and not curr_is_styled: + if prev_raw_tail.isalnum() and curr_raw_head.isalnum(): + return True + if ( + _classify_word_punctuation_boundary(prev_tail=prev_raw_tail, curr_head=curr_raw_head) + == InlineBoundary.SPACE + ): + return True + return False + + if prev_is_styled != curr_is_styled: + if len(curr_raw_text) > 3: + return True + if curr_raw_text.lower() in _COMMON_SHORT_WORDS: + return True + + return False + + +def _should_text_boundary_override_provenance( + *, + prev_item: TextItem, + item: TextItem, + text_boundary: InlineBoundary, + provenance_boundary: InlineBoundary, +) -> bool: + """Return whether text heuristics should beat provenance for this inline boundary.""" + if text_boundary == InlineBoundary.UNKNOWN: + return False + + if provenance_boundary == InlineBoundary.UNKNOWN: + return True + + if text_boundary == provenance_boundary: + return True + + if provenance_boundary == InlineBoundary.SPACE and text_boundary == InlineBoundary.JOIN: + return _is_ocr_safe_midword_join(prev_item=prev_item, item=item) + + if provenance_boundary == InlineBoundary.JOIN and text_boundary == InlineBoundary.SPACE: + return _is_false_ocr_word_join(prev_item=prev_item, item=item) + + return False + + class CommonParams(BaseModel): """Common serialization parameters.""" diff --git a/test/test_serialization.py b/test/test_serialization.py index dbb336289..b6911c27a 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -11,12 +11,18 @@ _DEFAULT_LABELS, InlineBoundary, _classify_ambiguous_word_boundary, + _classify_character_boundary, _classify_inline_boundary, _classify_provenance_boundary, _classify_text_boundary, + _is_false_ocr_word_join, + _is_markdown_syntax_join, + _is_ocr_safe_midword_join, _is_semantic_inline_atom, _is_styled_text, _join_inline_parts, + _should_provenance_join_override_spacing, + _should_text_boundary_override_provenance, create_ser_result, ) from docling_core.transforms.serializer.html import ( @@ -53,6 +59,7 @@ ProvenanceItem, RefItem, RichTableCell, + Script, SummaryMetaField, TableCell, TableData, @@ -1346,17 +1353,340 @@ def test_classify_source_boundary(prev_orig, curr_orig, prev_prov, curr_prov, ex assert _classify_provenance_boundary(prev_item=prev_item, item=curr_item) == expected -def test_classify_inline_boundary_handles_missing_items_and_whitespace(): +def _inline_boundary( + doc: DoclingDocument, + group, + *, + prev_text: str, + curr_text: str, + prev_rendered: str | None = None, + curr_rendered: str | None = None, + prev_formatting: Formatting | None = None, + curr_formatting: Formatting | None = None, + prev_hyperlink: str | None = None, + curr_hyperlink: str | None = None, + prev_orig: str | None = None, + curr_orig: str | None = None, + prev_prov: ProvenanceItem | None = None, + curr_prov: ProvenanceItem | None = None, +) -> InlineBoundary: + """Build adjacent inline items and classify their boundary.""" + prev_item = _add_inline_text( + doc, + group, + text=prev_text, + formatting=prev_formatting, + hyperlink=prev_hyperlink, + orig=prev_orig, + prov=prev_prov, + ) + curr_item = _add_inline_text( + doc, + group, + text=curr_text, + formatting=curr_formatting, + hyperlink=curr_hyperlink, + orig=curr_orig, + prov=curr_prov, + ) + return _classify_inline_boundary( + prev_text=prev_rendered or prev_text, + prev_item=prev_item, + text=curr_rendered or curr_text, + item=curr_item, + ) + + +def _join_inline_pair( + doc: DoclingDocument, + group, + *, + prev_text: str, + curr_text: str, + prev_rendered: str | None = None, + curr_rendered: str | None = None, + prev_formatting: Formatting | None = None, + curr_formatting: Formatting | None = None, + prev_orig: str | None = None, + curr_orig: str | None = None, + prev_prov: ProvenanceItem | None = None, + curr_prov: ProvenanceItem | None = None, +) -> str: + """Join two inline serialization parts with context-aware spacing.""" + prev_item = _add_inline_text( + doc, + group, + text=prev_text, + formatting=prev_formatting, + orig=prev_orig, + prov=prev_prov, + ) + curr_item = _add_inline_text( + doc, + group, + text=curr_text, + formatting=curr_formatting, + orig=curr_orig, + prov=curr_prov, + ) + return _join_inline_parts( + [ + create_ser_result(text=prev_rendered or prev_text, span_source=prev_item), + create_ser_result(text=curr_rendered or curr_text, span_source=curr_item), + ] + ) + + +_INLINE_BOUNDARY_CASES: dict[str, dict] = { + "ocr_false_gap_join": { + "prev_text": "Pars", + "curr_text": "ing", + "prev_orig": "Pars", + "curr_orig": "ing", + "curr_formatting": Formatting(italic=True), + "prev_prov": _make_prov(0, 4), + "curr_prov": _make_prov(6, 9), + "expected": InlineBoundary.JOIN, + }, + "ocr_false_join_space": { + "prev_text": "plain", + "curr_text": "text", + "prev_orig": "plain", + "curr_orig": "text", + "prev_prov": _make_prov(0, 5), + "curr_prov": _make_prov(5, 9), + "expected": InlineBoundary.SPACE, + }, + "styled_prefix_join": { + "prev_text": "D", + "curr_text": "ocling", + "prev_orig": "D", + "curr_orig": "ocling", + "prev_formatting": Formatting(bold=True), + "prev_prov": _make_prov(0, 1), + "curr_prov": _make_prov(3, 9), + "expected": InlineBoundary.JOIN, + }, + "provenance_fallback_space": { + "prev_text": "foo@", + "curr_text": "#bar", + "prev_orig": "foo@ ", + "curr_orig": "#bar", + "prev_prov": _make_prov(0, 4), + "curr_prov": _make_prov(6, 10), + "expected": InlineBoundary.SPACE, + "text_boundary_unknown": True, + }, + "provenance_spacing_bold_phrase": { + "prev_text": "bold (b)", + "curr_text": "example", + "prev_rendered": "**bold (b)**", + "prev_orig": "bold (b) ", + "curr_orig": "example", + "prev_formatting": Formatting(bold=True), + "prev_prov": _make_prov(0, 8), + "curr_prov": _make_prov(9, 16), + "expected": InlineBoundary.SPACE, + }, + "provenance_spacing_bold_and": { + "prev_text": "Bold", + "curr_text": "and", + "prev_rendered": "**Bold**", + "prev_orig": "Bold ", + "curr_orig": "and", + "prev_formatting": Formatting(bold=True), + "prev_prov": _make_prov(0, 4), + "curr_prov": _make_prov(5, 8), + "expected": InlineBoundary.SPACE, + }, + "provenance_spacing_underline_and": { + "prev_text": "underline", + "curr_text": "and", + "prev_orig": "underline ", + "curr_orig": "and", + "prev_formatting": Formatting(underline=True), + "prev_prov": _make_prov(0, 9), + "curr_prov": _make_prov(10, 13), + "expected": InlineBoundary.SPACE, + }, + "provenance_spacing_after_comma": { + "prev_text": "lake,", + "curr_text": "it's", + "prev_orig": "lake, ", + "curr_orig": "it's", + "prev_prov": _make_prov(0, 5), + "curr_prov": _make_prov(6, 10), + "expected": InlineBoundary.SPACE, + }, + "subscript_spacing_h2": { + "prev_text": "H", + "curr_text": "2", + "prev_orig": "H ", + "curr_orig": "2", + "curr_formatting": Formatting(script=Script.SUB), + "prev_prov": _make_prov(0, 1), + "curr_prov": _make_prov(2, 3), + "expected": InlineBoundary.SPACE, + }, + "subscript_spacing_o": { + "prev_text": "2", + "curr_text": "O", + "prev_orig": "2", + "curr_orig": "O", + "prev_formatting": Formatting(script=Script.SUB), + "prev_prov": _make_prov(2, 3), + "curr_prov": _make_prov(4, 5), + "expected": InlineBoundary.SPACE, + }, + "newline_source_spacing_text": { + "prev_text": "Text:", + "curr_text": "00:16.000 ----> 00:18.000", + "prev_orig": "Text:\n", + "curr_orig": "00:16.000 ----> 00:18.000", + "expected": InlineBoundary.SPACE, + }, + "newline_source_spacing_comma": { + "prev_text": "lake,", + "curr_text": "it's", + "prev_orig": "lake,\n", + "curr_orig": "it's", + "expected": InlineBoundary.SPACE, + }, + "citation_after_period": { + "prev_text": "hen.", + "curr_text": "[[ 3 ]]", + "expected": InlineBoundary.SPACE, + }, + "glossary_after_emphasis": { + "prev_text": "dūce", + "curr_text": "'diver'", + "prev_rendered": "*dūce*", + "prev_formatting": Formatting(italic=True), + "expected": InlineBoundary.SPACE, + }, + "markdown_punctuation_after_bold": { + "prev_text": "bold (b)", + "curr_text": ".", + "prev_rendered": "**bold (b)**", + "prev_formatting": Formatting(bold=True), + "expected": InlineBoundary.JOIN, + }, + "markdown_punctuation_after_link": { + "prev_text": "Example", + "curr_text": ".", + "prev_rendered": "[Example](https://example.com/)", + "prev_hyperlink": "https://example.com/", + "expected": InlineBoundary.JOIN, + }, + "ampersand_spacing": { + "prev_text": "00:18.000", + "curr_text": "&", + "expected": InlineBoundary.SPACE, + }, + "adjacent_links": { + "prev_text": "[[ 3 ]]", + "curr_text": "[[ 4 ]]", + "prev_rendered": "[[ 3 ]](#cite_note-3)", + "prev_hyperlink": "#cite_note-3", + "curr_hyperlink": "#cite_note-4", + "expected": InlineBoundary.SPACE, + }, + "provenance_join_blocked_by_word_punct": { + "prev_text": "hen.", + "curr_text": "[[ 3 ]]", + "prev_orig": "hen.", + "curr_orig": "[[ 3 ]]", + "prev_prov": _make_prov(0, 4), + "curr_prov": _make_prov(4, 11), + "expected": InlineBoundary.SPACE, + }, + "styled_prefix_common_word_space": { + "prev_text": "D", + "curr_text": "and", + "prev_orig": "D", + "curr_orig": "and", + "prev_formatting": Formatting(bold=True), + "prev_prov": _make_prov(0, 1), + "curr_prov": _make_prov(3, 6), + "expected": InlineBoundary.SPACE, + }, + "provenance_join_blocked_by_emphasis_quote": { + "prev_text": "dūce", + "curr_text": "'diver'", + "prev_rendered": "*dūce*", + "prev_formatting": Formatting(italic=True), + "prev_orig": "dūce", + "curr_orig": "'diver'", + "prev_prov": _make_prov(0, 4), + "curr_prov": _make_prov(4, 11), + "expected": InlineBoundary.SPACE, + }, +} + + +@pytest.mark.parametrize("case_id", _INLINE_BOUNDARY_CASES.keys()) +def test_classify_inline_boundary_cases(case_id: str): + case = _INLINE_BOUNDARY_CASES[case_id] + doc, group = _make_inline_doc() + expected = case["expected"] + + if case.get("text_boundary_unknown"): + prev_item = _add_inline_text( + doc, + group, + text=case["prev_text"], + orig=case.get("prev_orig"), + prov=case.get("prev_prov"), + ) + curr_item = _add_inline_text( + doc, + group, + text=case["curr_text"], + orig=case.get("curr_orig"), + prov=case.get("curr_prov"), + ) + assert _classify_text_boundary(prev_item=prev_item, item=curr_item) == InlineBoundary.UNKNOWN + + assert _inline_boundary(doc, group, **{k: v for k, v in case.items() if k != "expected" and k != "text_boundary_unknown"}) == expected + + +def test_classify_inline_boundary_without_items(): assert ( _classify_inline_boundary(prev_text="foo ", prev_item=None, text="bar", item=None) == InlineBoundary.JOIN ) assert ( _classify_inline_boundary(prev_text="foo", prev_item=None, text="bar", item=None) - == InlineBoundary.UNKNOWN + == InlineBoundary.SPACE ) +_JOIN_INLINE_CASES: dict[str, dict] = { + "provenance_spacing": { + "prev_text": "Text:", + "curr_text": "00:16.000 ----> 00:18.000", + "prev_orig": "Text:\n", + "curr_orig": "00:16.000 ----> 00:18.000", + "expected": "Text: 00:16.000 ----> 00:18.000", + }, + "punctuation_join": { + "prev_text": "hello-", + "curr_text": "world", + "curr_formatting": Formatting(italic=True), + "expected": "hello-world", + }, +} + + +@pytest.mark.parametrize("case_id", _JOIN_INLINE_CASES.keys()) +def test_join_inline_parts_spacing_cases(case_id: str): + case = _JOIN_INLINE_CASES[case_id] + doc, group = _make_inline_doc() + expected = case["expected"] + join_kwargs = {k: v for k, v in case.items() if k not in {"expected"}} + assert _join_inline_pair(doc, group, **join_kwargs) == expected + + @pytest.mark.parametrize( ("prev_text", "prev_formatting", "prev_hyperlink", "curr_text", "curr_formatting", "curr_hyperlink", "expected"), [ @@ -1367,7 +1697,19 @@ def test_classify_inline_boundary_handles_missing_items_and_whitespace(): ("strong emphasis", Formatting(bold=True), None, "tail", None, None, InlineBoundary.SPACE), ("bold", Formatting(bold=True), None, "italic", Formatting(italic=True), None, InlineBoundary.SPACE), ("bold", Formatting(bold=True), None, "&", None, None, InlineBoundary.SPACE), - ("hello-", None, None, "world", Formatting(italic=True), None, InlineBoundary.UNKNOWN), + ("hello-", None, None, "world", Formatting(italic=True), None, InlineBoundary.JOIN), + ("foo/", None, None, "bar", None, None, InlineBoundary.JOIN), + ("word", None, None, ")", None, None, InlineBoundary.JOIN), + ("hypotenuse", None, None, "c", Formatting(italic=True), None, InlineBoundary.SPACE), + ("plain", None, None, "text", None, None, InlineBoundary.SPACE), + ("&", None, None, "word", Formatting(italic=True), None, InlineBoundary.SPACE), + ("x", Formatting(script=Script.SUB), None, "+", None, None, InlineBoundary.SPACE), + ("foo,", None, None, "b", Formatting(bold=True), None, InlineBoundary.SPACE), + ("D", Formatting(bold=True), None, "and", None, None, InlineBoundary.SPACE), + ("item)", None, None, "[note", None, None, InlineBoundary.SPACE), + ("num", None, None, "&", None, None, InlineBoundary.SPACE), + ("'", None, None, "'", None, None, InlineBoundary.JOIN), + ("-", None, None, "/", None, None, InlineBoundary.JOIN), ], ) def test_classify_text_boundary( @@ -1458,3 +1800,194 @@ def test_common_inline_helper_flags(): ) def test_classify_ambiguous_word_boundary(curr_raw_text, expected): assert _classify_ambiguous_word_boundary(curr_raw_text=curr_raw_text) == expected + + +@pytest.mark.parametrize( + ("prev_tail", "curr_head", "prev_text", "text", "expected"), + [ + (".", ")", "", "", True), + ("-", "w", "", "", True), + ("(", "w", "", "", True), + ("w", "/", "", "", True), + ("'", "'", "", "", True), + ("_", ".", "_", "", True), + ("*", ".", "_*", "", True), + ("o", "*", "", "*word", False), + ("x", "y", "", "", False), + ], +) +def test_is_markdown_syntax_join(prev_tail, curr_head, prev_text, text, expected): + assert ( + _is_markdown_syntax_join( + prev_tail=prev_tail, + curr_head=curr_head, + prev_text=prev_text, + text=text, + ) + == expected + ) + + +def test_inline_boundary_text_to_code_rendered_fallback(): + doc, group = _make_inline_doc() + prev_item = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="see!") + code_item = doc.add_code(text="code", parent=group, hyperlink="#link") + assert ( + _classify_inline_boundary( + prev_text="see!", + prev_item=prev_item, + text="[`code`](#link)", + item=code_item, + ) + == InlineBoundary.UNKNOWN + ) + + +def test_inline_boundary_semantic_to_text_word_join(): + doc, group = _make_inline_doc() + formula_item = doc.add_formula(text="x/y", parent=group) + next_item = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="/more") + assert ( + _classify_inline_boundary( + prev_text="$x/y$", + prev_item=formula_item, + text="/more", + item=next_item, + ) + == InlineBoundary.JOIN + ) + + +def test_inline_boundary_helper_branches(): + doc, group = _make_inline_doc() + + prev_hen = _add_inline_text(doc, group, text="hen.", orig="hen.") + curr_citation = _add_inline_text(doc, group, text="[[ 3 ]]") + assert ( + _should_provenance_join_override_spacing( + prev_item=prev_hen, + item=curr_citation, + prev_text="hen.", + text="[[ 3 ]]", + ) + is False + ) + + prev_emphasis = _add_inline_text( + doc, + group, + text="dūce", + formatting=Formatting(italic=True), + ) + curr_gloss = _add_inline_text(doc, group, text="'diver'") + assert ( + _should_provenance_join_override_spacing( + prev_item=prev_emphasis, + item=curr_gloss, + prev_text="*dūce*", + text="'diver'", + ) + is False + ) + + prev_pars = _add_inline_text(doc, group, text="Pars", orig="Pars", prov=_make_prov(0, 4)) + curr_ing = _add_inline_text( + doc, + group, + text="ing", + orig="ing", + formatting=Formatting(italic=True), + prov=_make_prov(6, 9), + ) + assert _is_ocr_safe_midword_join(prev_item=prev_pars, item=curr_ing) is True + + prev_h = _add_inline_text(doc, group, text="H") + curr_sub = _add_inline_text(doc, group, text="2", formatting=Formatting(script=Script.SUB)) + assert _is_ocr_safe_midword_join(prev_item=prev_h, item=curr_sub) is False + assert _is_false_ocr_word_join(prev_item=prev_h, item=curr_sub) is False + + prev_plain = _add_inline_text(doc, group, text="plain") + curr_text = _add_inline_text(doc, group, text="text") + assert _is_false_ocr_word_join(prev_item=prev_plain, item=curr_text) is True + + prev_lake = _add_inline_text(doc, group, text="lake,") + curr_its = _add_inline_text(doc, group, text="it's") + assert _is_false_ocr_word_join(prev_item=prev_lake, item=curr_its) is True + + prev_word = _add_inline_text(doc, group, text="foo@") + curr_hash = _add_inline_text(doc, group, text="#bar") + assert _is_false_ocr_word_join(prev_item=prev_word, item=curr_hash) is False + + prev_bold = _add_inline_text(doc, group, text="bold", formatting=Formatting(bold=True)) + curr_tail = _add_inline_text(doc, group, text="tails") + assert _is_false_ocr_word_join(prev_item=prev_bold, item=curr_tail) is True + + assert ( + _should_text_boundary_override_provenance( + prev_item=prev_word, + item=curr_hash, + text_boundary=InlineBoundary.SPACE, + provenance_boundary=InlineBoundary.JOIN, + ) + is False + ) + + prev_bold_d = _add_inline_text(doc, group, text="D", formatting=Formatting(bold=True)) + curr_and = _add_inline_text(doc, group, text="and") + assert _is_false_ocr_word_join(prev_item=prev_bold_d, item=curr_and) is True + + prev_bold_d = _add_inline_text(doc, group, text="D", formatting=Formatting(bold=True)) + curr_x = _add_inline_text(doc, group, text="x", formatting=Formatting(italic=True)) + assert ( + _should_text_boundary_override_provenance( + prev_item=prev_bold_d, + item=curr_x, + text_boundary=InlineBoundary.SPACE, + provenance_boundary=InlineBoundary.JOIN, + ) + is False + ) + + prev_bold_phrase = _add_inline_text( + doc, + group, + text="bold (b)", + formatting=Formatting(bold=True), + ) + curr_period = _add_inline_text(doc, group, text=".") + assert ( + _should_provenance_join_override_spacing( + prev_item=prev_bold_phrase, + item=curr_period, + prev_text="**bold (b)**", + text=".", + ) + is True + ) + + assert ( + _should_provenance_join_override_spacing( + prev_item=prev_pars, + item=curr_ing, + prev_text="Pars", + text="ing", + ) + is True + ) + + code_item = doc.add_code(text="code", parent=group) + text_item = _add_inline_text(doc, group, text="tail") + assert ( + _should_provenance_join_override_spacing( + prev_item=code_item, + item=text_item, + prev_text="code", + text="tail", + ) + is True + ) + + +def test_classify_character_boundary_empty_inputs(): + assert _classify_character_boundary(prev_tail="", curr_head="a") == InlineBoundary.UNKNOWN + assert _classify_character_boundary(prev_tail="a", curr_head="") == InlineBoundary.UNKNOWN From f5298f893fd4e9dd63a0e304dd26c0180021b0b5 Mon Sep 17 00:00:00 2001 From: wanadzhar913 Date: Tue, 23 Jun 2026 19:05:09 +0800 Subject: [PATCH 3/3] fix: formatting issues Signed-off-by: wanadzhar913 --- test/test_serialization.py | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/test/test_serialization.py b/test/test_serialization.py index b6911c27a..a562efedc 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -41,14 +41,12 @@ ) from docling_core.transforms.serializer.webvtt import WebVTTDocSerializer, WebVTTParams from docling_core.transforms.visualizer.layout_visualizer import LayoutVisualizer - from docling_core.types.doc import DoclingDocument from docling_core.types.doc.base import BoundingBox, ImageRefMode from docling_core.types.doc.document import ( BaseMeta, CharSpan, DescriptionAnnotation, - DoclingDocument, EntitiesMetaField, EntityMention, Formatting, @@ -1231,9 +1229,7 @@ def test_html_inline_group_no_spaces(): ) # This should serialize as Projecting without space - ser = HTMLDocSerializer( - doc=doc, params=HTMLParams(html_head="", prettify=False) - ) + ser = HTMLDocSerializer(doc=doc, params=HTMLParams(html_head="", prettify=False)) actual = ser.serialize().text body_content = _extract_body_content(actual) @@ -1283,9 +1279,7 @@ def test_html_inline_group_mixed_formatting_mid_word(): ), ) - ser = HTMLDocSerializer( - doc=doc, params=HTMLParams(html_head="", prettify=False) - ) + ser = HTMLDocSerializer(doc=doc, params=HTMLParams(html_head="", prettify=False)) actual = ser.serialize().text body_content = _extract_body_content(actual) @@ -1309,9 +1303,7 @@ def test_html_inline_group_single_part(): doc, group = _make_inline_doc() _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="Single") - ser = HTMLDocSerializer( - doc=doc, params=HTMLParams(html_head="", prettify=False) - ) + ser = HTMLDocSerializer(doc=doc, params=HTMLParams(html_head="", prettify=False)) actual = ser.serialize().text body_content = _extract_body_content(actual) @@ -1647,18 +1639,17 @@ def test_classify_inline_boundary_cases(case_id: str): ) assert _classify_text_boundary(prev_item=prev_item, item=curr_item) == InlineBoundary.UNKNOWN - assert _inline_boundary(doc, group, **{k: v for k, v in case.items() if k != "expected" and k != "text_boundary_unknown"}) == expected + assert ( + _inline_boundary( + doc, group, **{k: v for k, v in case.items() if k != "expected" and k != "text_boundary_unknown"} + ) + == expected + ) def test_classify_inline_boundary_without_items(): - assert ( - _classify_inline_boundary(prev_text="foo ", prev_item=None, text="bar", item=None) - == InlineBoundary.JOIN - ) - assert ( - _classify_inline_boundary(prev_text="foo", prev_item=None, text="bar", item=None) - == InlineBoundary.SPACE - ) + assert _classify_inline_boundary(prev_text="foo ", prev_item=None, text="bar", item=None) == InlineBoundary.JOIN + assert _classify_inline_boundary(prev_text="foo", prev_item=None, text="bar", item=None) == InlineBoundary.SPACE _JOIN_INLINE_CASES: dict[str, dict] = { @@ -1773,9 +1764,7 @@ def test_common_inline_helper_flags(): doc, group = _make_inline_doc() plain = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="plain") bold = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="bold", formatting=Formatting(bold=True)) - linked = _add_inline_text( - doc, group, label=DocItemLabel.TEXT, text="link", hyperlink="https://example.com" - ) + linked = _add_inline_text(doc, group, label=DocItemLabel.TEXT, text="link", hyperlink="https://example.com") code = doc.add_code(text="print()", parent=group) formula = doc.add_formula(text="E=mc^2", parent=group)