Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 60 additions & 1 deletion app/pybind_parse.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -908,7 +908,66 @@ PYBIND11_MODULE(pdf_parsers, m) {

Returns:
dict: A None or string of the metadata in xml of the document.)")


.def("get_attachments",
[](docling::docling_parser &self, const std::string &key) -> nlohmann::json {
return self.get_attachments(key);
},
pybind11::arg("key"),
R"(
Retrieve attachments for the document identified by its unique key.

Parameters:
key (str): The unique key of the document.

Returns:
list: A JSON array of attachment metadata (name, mime_type, size, annotations).

Raises:
RuntimeError: If the key is not loaded.)")

.def("get_attachment_data",
[](docling::docling_parser &self, const std::string &key, int index, long long max_size) -> pybind11::bytes {
return self.get_attachment_data(key, index, max_size);
},
pybind11::arg("key"),
pybind11::arg("index"),
pybind11::arg("max_size"),
R"(
Retrieve raw bytes for a single attachment.

Parameters:
key (str): The unique key of the document.
index (int): Attachment index from get_attachments().
max_size (int): Maximum allowed decoded size in bytes (required).

Returns:
bytes: Decoded attachment payload.

Raises:
RuntimeError: If the key is not loaded, index is out of range,
size exceeds max_size, or the stream cannot be decoded.)")

.def("write_attachment_data",
[](docling::docling_parser &self, const std::string &key, int index, long long max_size, const std::string &path) {
self.write_attachment_data(key, index, max_size, path);
},
pybind11::arg("key"),
pybind11::arg("index"),
pybind11::arg("max_size"),
pybind11::arg("path"),
R"(
Stream a single attachment directly to a file without buffering the full decoded payload.

Parameters:
key (str): The unique key of the document.
index (int): Attachment index from get_attachments().
max_size (int): Maximum allowed decoded size in bytes (required).
path (str): Destination file path.

Raises:
RuntimeError: If size exceeds max_size or the stream cannot be decoded.)")

.def("get_page_decoder",
[](docling::docling_parser &self,
const std::string &key,
Expand Down
184 changes: 183 additions & 1 deletion docling_parse/pdf_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
import hashlib
import logging
import math
import os
import tempfile
from enum import IntEnum
from io import BytesIO
from pathlib import Path
from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union
from typing import Any, BinaryIO, Dict, Iterator, List, Optional, Sequence, Tuple, Union

from docling_core.types.doc.base import BoundingBox, CoordOrigin, ImageRefMode
from docling_core.types.doc.document import ImageRef
Expand All @@ -15,6 +17,8 @@
BoundingRectangle,
ColorRGBA,
Coord2D,
FileAttachmentAnnotation,
PdfAttachment,
PdfHyperlink,
PdfMetaData,
PdfPageBoundaryType,
Expand Down Expand Up @@ -77,6 +81,46 @@
_log = logging.getLogger(__name__)


class _AttachmentDeletingFile:
"""File wrapper that unlinks its mkstemp path on close.

Created once per large attachment; defined at module scope so the
class object is not re-created on every get_attachment_stream call.
"""

def __init__(self, fd: int, path: str):
self._path = path
self._file = os.fdopen(fd, "w+b")
self.name = path # for test compat

def read(self, *a, **kw):
return self._file.read(*a, **kw)

def seek(self, *a, **kw):
return self._file.seek(*a, **kw)

def tell(self):
return self._file.tell()

def close(self):
try:
self._file.close()
finally:
try:
os.unlink(self._path)
except Exception:
pass

def __getattr__(self, n):
return getattr(self._file, n)

def __enter__(self):
return self

def __exit__(self, *a):
self.close()


class PdfTocEntry(BaseModel):
"""PDF table of contents entry (recursive structure).

Expand Down Expand Up @@ -900,6 +944,144 @@ def get_annotations(self) -> PdfAnnotations | None:
else:
raise RuntimeError("This document is not loaded.")

def get_attachments(self) -> List[PdfAttachment]:
"""Get attachment metadata (does not decode bytes).

Returns:
List[PdfAttachment]: metadata with name, mime_type, size, and page annotations.

Raises:
RuntimeError: if the document is not loaded (missing key).
"""
if not self.is_loaded():
raise RuntimeError("This document is not loaded.")

raw = self._parser.get_attachments(key=self._key)
# C++ get_attachments now throws on missing key, consistent with
# get_attachment_data — empty doc returns [].
result: List[PdfAttachment] = []
for item in raw:
name = item.get("name", "attachment")
mime = item.get("mime_type")
size = int(item.get("size", 0) or 0)
anns: List[FileAttachmentAnnotation] = []
for ja in item.get("annotations") or []:
try:
bbox_vals = ja.get("bbox") or [0, 0, 0, 0]
bbox = _to_bounding_rectangle(tuple(float(v) for v in bbox_vals)) # type: ignore[arg-type]
anns.append(
FileAttachmentAnnotation(
page_no=int(ja.get("page_no", 0)) + 1, bbox=bbox
)
)
except Exception:
_log.debug("Skipping malformed annotation %s", ja, exc_info=True)
result.append(
PdfAttachment(name=name, mime_type=mime, size=size, annotations=anns)
)
return result

def get_attachment_data(self, index: int, *, max_size: int) -> bytes:
"""Get raw bytes for a single attachment (memory-safe, requires max_size).

Args:
index: Attachment index from get_attachments().
max_size: Maximum allowed decoded size (required, enforces caller policy).

Raises:
ValueError/RuntimeError: if size exceeds max_size or decode fails.
"""
if not self.is_loaded():
raise RuntimeError("This document is not loaded.")
if not isinstance(max_size, int):
raise TypeError("max_size is required and must be an int")
return bytes(
self._parser.get_attachment_data(
key=self._key, index=index, max_size=max_size
)
)

# Attachments at or below this size are served from memory; larger ones
# are spooled to a temp file that deletes itself on close.
_ATTACHMENT_MEMORY_LIMIT_BYTES = 8 * 1024 * 1024

def get_attachment_stream(self, index: int, *, max_size: int) -> BinaryIO:
"""Get a readable BinaryIO for one attachment (spills to temp file if large).

Args:
index: Attachment index from get_attachments().
max_size: Maximum allowed decoded size (required).

Returns:
BinaryIO: for size <= 8 MB a BytesIO; otherwise an open file
positioned at 0 that deletes itself on close (via
mkstemp wrapper). Caller must close() when done.

Raises:
ValueError/RuntimeError: if size exceeds max_size or decode fails.
"""
if not self.is_loaded():
raise RuntimeError("This document is not loaded.")
if not isinstance(max_size, int):
raise TypeError("max_size is required and must be an int")

# Use metadata to decide transport without decoding first.
# size==0 means unknown — fall back to max_size.
try:
atts = self.get_attachments()
meta_size = atts[index].size if 0 <= index < len(atts) else 0
except Exception:
meta_size = 0
effective_size = meta_size if meta_size > 0 else max_size

if effective_size > max_size:
raise RuntimeError(
f"attachment size {effective_size} exceeds max_size {max_size}"
)

use_memory = effective_size <= self._ATTACHMENT_MEMORY_LIMIT_BYTES
# Also cap memory by max_size
if max_size <= self._ATTACHMENT_MEMORY_LIMIT_BYTES:
use_memory = True

if use_memory:
data = self.get_attachment_data(index, max_size=max_size)
if len(data) <= self._ATTACHMENT_MEMORY_LIMIT_BYTES:
return BytesIO(data)
# Size was underestimated — fall through to file path
# but avoid double copy: spill via pipe if available
effective_size = len(data)
del data # free before file spill

# Large path: stream directly to file without holding full bytes in memory.
# Prefer native pipe if available (avoids C++ Buffer allocation).
if hasattr(self._parser, "write_attachment_data"):
tmp_fd, tmp_path = tempfile.mkstemp(suffix="__attachment")
wrapper = _AttachmentDeletingFile(tmp_fd, tmp_path)
try:
self._parser.write_attachment_data(
key=self._key, index=index, max_size=max_size, path=tmp_path
)
wrapper.seek(0)
return wrapper # type: ignore[return-value]
except Exception:
wrapper.close()
raise
else:
# Fallback: legacy path — still avoid 2x memory by deleting after write
data = self.get_attachment_data(index, max_size=max_size)
tmp = tempfile.NamedTemporaryFile(delete=True, suffix="__attachment")
try:
tmp.write(data)
tmp.flush()
tmp.seek(0)
return tmp
except Exception:
tmp.close()
raise
finally:
del data

def get_page(
self,
page_no: int,
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ requires-python = ">=3.10"
dependencies = [
"pillow>=10.0.0,<13.0.0",
"pydantic>=2.0.0",
"docling-core>=2.85.0,<3.0.0",
"docling-core>=2.91.0,<3.0.0",
"pywin32>=305; sys_platform == 'win32'",
]
[project.urls]
Expand Down
Loading
Loading