From b361bb7ebebc14cbe159be27601b26b254f6b2f7 Mon Sep 17 00:00:00 2001 From: Max Hsu Date: Sat, 18 Jul 2026 18:46:49 +0800 Subject: [PATCH] feat(markdown): include table image in Markdown export when available MarkdownTableSerializer previously ignored TableItem.image entirely, only emitting the text grid, unlike MarkdownPictureSerializer which honors image_mode for pictures. Extract the shared image-emission logic into a module-level _serialize_image_part() usable by both serializers, and move _image_to_base64 up from PictureItem to the shared FloatingItem base so TableItem can call it too. MarkdownTableSerializer now appends the table image alongside the text table when item.image is set and image_mode is not PLACEHOLDER. Tables without a generated image (the common case) see no output change. Reworked onto the post-#664 module split (items/node.py, items/picture, transforms/serializer/markdown) after the original branch went stale. Closes #474 Signed-off-by: Max Hsu --- .../transforms/serializer/markdown.py | 99 ++++++++++--------- docling_core/types/doc/items/node.py | 11 +++ .../types/doc/items/picture/picture.py | 11 --- test/test_serialization.py | 79 +++++++++++++++ 4 files changed, 143 insertions(+), 57 deletions(-) diff --git a/docling_core/transforms/serializer/markdown.py b/docling_core/transforms/serializer/markdown.py index 955fd080b..aaa0418e2 100644 --- a/docling_core/transforms/serializer/markdown.py +++ b/docling_core/transforms/serializer/markdown.py @@ -427,6 +427,46 @@ def serialize( ) +def _serialize_image_part( + item: FloatingItem, + doc: DoclingDocument, + image_mode: ImageRefMode, + image_placeholder: str, + **kwargs: Any, +) -> SerializationResult: + """Render a floating item's image per the image mode. + + Shared by the picture and table serializers so both honor image_mode + identically for any FloatingItem that carries an image. + """ + error_response = "" + if image_mode == ImageRefMode.PLACEHOLDER: + text_res = image_placeholder + elif image_mode == ImageRefMode.EMBEDDED: + # short-cut: we already have the image in base64 + if isinstance(item.image, ImageRef) and isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data": + text_res = f"![Image]({item.image.uri})" + else: + # get the item.image._pil or crop it out of the page-image + img = item.get_image(doc=doc) + if img is not None: + imgb64 = item._image_to_base64(img) + text_res = f"![Image](data:image/png;base64,{imgb64})" + else: + text_res = error_response + elif image_mode == ImageRefMode.REFERENCED: + if not isinstance(item.image, ImageRef) or ( + isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data" + ): + text_res = image_placeholder + else: + text_res = f"![Image]({item.image.uri!s})" + else: + text_res = image_placeholder + + return create_ser_result(text=text_res, span_source=item) + + class MarkdownTableSerializer(BaseTableSerializer): """Markdown-specific table item serializer.""" @@ -584,6 +624,18 @@ def serialize( if table_text: res_parts.append(create_ser_result(text=table_text, span_source=item)) + # Emit the table image alongside the text grid when one was generated. + # Tables without an image (the common case) stay a strict no-op. + if item.image is not None and params.image_mode != ImageRefMode.PLACEHOLDER: + img_res = _serialize_image_part( + item=item, + doc=doc, + image_mode=params.image_mode, + image_placeholder=params.image_placeholder, + ) + if img_res.text: + res_parts.append(img_res) + text_res = "\n\n".join([r.text for r in res_parts]) return create_ser_result(text=text_res, span_source=res_parts) @@ -622,7 +674,7 @@ def serialize( if ann_res.text: res_parts.append(ann_res) - img_res = self._serialize_image_part( + img_res = _serialize_image_part( item=item, doc=doc, image_mode=params.image_mode, @@ -648,51 +700,6 @@ def serialize( return create_ser_result(text=text_res, span_source=res_parts) - def _serialize_image_part( - self, - item: PictureItem, - doc: DoclingDocument, - image_mode: ImageRefMode, - image_placeholder: str, - **kwargs: Any, - ) -> SerializationResult: - error_response = ( - "" - ) - if image_mode == ImageRefMode.PLACEHOLDER: - text_res = image_placeholder - elif image_mode == ImageRefMode.EMBEDDED: - # short-cut: we already have the image in base64 - if ( - isinstance(item.image, ImageRef) - and isinstance(item.image.uri, AnyUrl) - and item.image.uri.scheme == "data" - ): - text = f"![Image]({item.image.uri})" - text_res = text - else: - # get the item.image._pil or crop it out of the page-image - img = item.get_image(doc=doc) - - if img is not None: - imgb64 = item._image_to_base64(img) - text = f"![Image](data:image/png;base64,{imgb64})" - - text_res = text - else: - text_res = error_response - elif image_mode == ImageRefMode.REFERENCED: - if not isinstance(item.image, ImageRef) or ( - isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data" - ): - text_res = image_placeholder - else: - text_res = f"![Image]({item.image.uri!s})" - else: - text_res = image_placeholder - - return create_ser_result(text=text_res, span_source=item) - class MarkdownKeyValueSerializer(BaseKeyValueSerializer): """Markdown-specific key-value item serializer.""" diff --git a/docling_core/types/doc/items/node.py b/docling_core/types/doc/items/node.py index 54f68b02f..60e52ad49 100644 --- a/docling_core/types/doc/items/node.py +++ b/docling_core/types/doc/items/node.py @@ -1,6 +1,8 @@ """Base document-tree node items: NodeItem, DocItem, FloatingItem.""" +import base64 from collections.abc import Sequence +from io import BytesIO from typing import TYPE_CHECKING, Annotated, Optional from PIL import Image as PILImage @@ -231,3 +233,12 @@ def get_image(self, doc: "DoclingDocument", prov_index: int = 0) -> Optional[PIL if self.image is not None: return self.image.pil_image return super().get_image(doc=doc, prov_index=prov_index) + + # Convert the image to Base64 + def _image_to_base64(self, pil_image, format="PNG"): + """Base64 representation of the image.""" + buffered = BytesIO() + pil_image.save(buffered, format=format) # Save the image to the byte stream + img_bytes = buffered.getvalue() # Get the byte data + img_base64 = base64.b64encode(img_bytes).decode("utf-8") # Encode to Base64 and decode to string + return img_base64 diff --git a/docling_core/types/doc/items/picture/picture.py b/docling_core/types/doc/items/picture/picture.py index 924c5c2c9..06a14d69c 100644 --- a/docling_core/types/doc/items/picture/picture.py +++ b/docling_core/types/doc/items/picture/picture.py @@ -1,12 +1,10 @@ """Picture item and its annotation-data union.""" -import base64 import hashlib import logging import typing import warnings from collections.abc import Sequence -from io import BytesIO from typing import TYPE_CHECKING, Annotated, Optional, Union from PIL import Image as PILImage @@ -143,15 +141,6 @@ def _migrate_annotations_to_meta(self) -> Self: return self - # Convert the image to Base64 - def _image_to_base64(self, pil_image, format="PNG"): - """Base64 representation of the image.""" - buffered = BytesIO() - pil_image.save(buffered, format=format) # Save the image to the byte stream - img_bytes = buffered.getvalue() # Get the byte data - img_base64 = base64.b64encode(img_bytes).decode("utf-8") # Encode to Base64 and decode to string - return img_base64 - @staticmethod def _image_to_hexhash(img: Optional[PILImage.Image]) -> Optional[str]: """Hexash from the image.""" diff --git a/test/test_serialization.py b/test/test_serialization.py index b6bd4b1db..9f6ecca35 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -6,6 +6,7 @@ from xml.etree import ElementTree as ET import pytest +from PIL import Image from docling_core.transforms.serializer.common import _DEFAULT_LABELS from docling_core.transforms.serializer.html import ( @@ -32,12 +33,14 @@ DescriptionAnnotation, EntitiesMetaField, EntityMention, + ImageRef, LanguageMetaField, PictureClassificationMetaField, PictureClassificationPrediction, PictureMeta, RefItem, RichTableCell, + Size, SummaryMetaField, TableCell, TableData, @@ -352,6 +355,82 @@ def test_md_single_row_table(): verify(exp_file=exp_file, actual=actual) +def test_md_table_with_image_embedded(): + doc = DoclingDocument(name="") + table = doc.add_table(data=TableData(num_rows=1, num_cols=1)) + doc.add_table_cell( + table_item=table, + cell=TableCell( + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=0, + end_col_offset_idx=1, + text="foo", + ), + ) + table.image = ImageRef.from_pil(image=Image.new("RGB", (2, 2)), dpi=72) + + ser = MarkdownDocSerializer( + doc=doc, + params=MarkdownParams(image_mode=ImageRefMode.EMBEDDED), + ) + actual = ser.serialize().text + assert "| foo |" in actual + assert "![Image](data:image/png;base64," in actual + + +def test_md_table_with_image_referenced(): + doc = DoclingDocument(name="") + table = doc.add_table(data=TableData(num_rows=1, num_cols=1)) + doc.add_table_cell( + table_item=table, + cell=TableCell( + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=0, + end_col_offset_idx=1, + text="foo", + ), + ) + table.image = ImageRef( + mimetype="image/png", + dpi=72, + size=Size(width=2, height=2), + uri="table_0.png", + ) + + ser = MarkdownDocSerializer( + doc=doc, + params=MarkdownParams(image_mode=ImageRefMode.REFERENCED), + ) + actual = ser.serialize().text + assert actual == "| foo |\n|-------|\n\n![Image](table_0.png)" + + +def test_md_table_without_image_unaffected_by_image_mode(): + """A table with no generated image must not gain an image line, even if + image_mode is set — this must remain a strict no-op for the common case.""" + doc = DoclingDocument(name="") + table = doc.add_table(data=TableData(num_rows=1, num_cols=1)) + doc.add_table_cell( + table_item=table, + cell=TableCell( + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=0, + end_col_offset_idx=1, + text="foo", + ), + ) + + ser = MarkdownDocSerializer( + doc=doc, + params=MarkdownParams(image_mode=ImageRefMode.EMBEDDED), + ) + actual = ser.serialize().text + assert actual == "| foo |\n|-------|" + + def test_md_pipe_in_table(): doc = DoclingDocument(name="Pipe in Table") table = doc.add_table(data=TableData(num_rows=1, num_cols=1))