feat(LAB-4612): support polygon annotations in PDF projects

docs/sdk/label_parsing.md

@@ -150,6 +150,45 @@ label.jobs["BBOX_JOB"].annotations.content
 label.jobs["BBOX_JOB"].bounding_poly_annotations.content
 ```
 
+#### Object detection in PDFs
+
+In a PDF project, the geometry of a bounding box or of a polygon is not on the annotation itself but one level deeper, in a nested `.annotations` list: a PDF page has its own coordinate system, so the vertices are normalized to the page named by `.page_number_array`.
+
+##### `.type`
+
+Returns the tool used to draw the annotation, `"rectangle"` or `"polygon"`.
+
+```python
+label.jobs["DETECTION_JOB"].annotations[0].type
+```
+
+PDF labels only carry the tool when it is not the rectangle, so an annotation without it is reported as a `"rectangle"`. This is a read-time default only: it is never written back, and exporting a label leaves it unchanged.
+
+##### `.annotations`
+
+Returns the page-level positions of the annotation. A polygon always has exactly one, since a polygon lives on a single page.
+
+```python
+label.jobs["DETECTION_JOB"].annotations[0].annotations[0]
+```
+
+##### `.polys` and `.bounding_poly`
+
+Return the vertices of the shape. For a polygon both hold the same ring of `N >= 3` vertices.
+
+```python
+label.jobs["DETECTION_JOB"].annotations[0].annotations[0].polys
+label.jobs["DETECTION_JOB"].annotations[0].annotations[0].bounding_poly[0].normalized_vertices
+```
+
+##### `.page_number_array`
+
+Returns the page the annotation is drawn on, 1-indexed.
+
+```python
+label.jobs["DETECTION_JOB"].annotations[0].annotations[0].page_number_array
+```
+
 #### Point detection
 
 ##### `.point`

src/kili/services/label_data_parsing/annotation.py

@@ -5,6 +5,7 @@
 from collections.abc import Iterator, Sequence
 from typing import Literal, Optional, Union
 
+from kili_formats.types import Job
 from typeguard import typechecked
 
 from kili.services.label_data_parsing import category as category_module
@@ -17,6 +18,11 @@
 from .utils import get_children_job_names
 
 
+def _is_pdf_object_detection(project_info: Project, job_interface: Job) -> bool:
+    """Return whether the annotation belongs to an object detection job of a PDF project."""
+    return project_info["inputType"] == "PDF" and job_interface["mlTask"] == "OBJECT_DETECTION"
+
+
 class _BaseAnnotation:
     """Class for parsing the "annotations" key of a job response.
 
@@ -73,8 +79,14 @@ def as_dict(self) -> dict:
         ret = {
             k: v
             for k, v in self._json_data.items()
-            if k not in ("categories", "children", "boundingPoly", "points")
+            if k not in ("categories", "children", "boundingPoly", "points", "annotations")
         }
+        if "annotations" in self._json_data:
+            ret["annotations"] = (
+                self._json_data["annotations"]
+                if isinstance(self._json_data["annotations"], list)
+                else self._json_data["annotations"].as_list()
+            )
         if "categories" in self._json_data:
             ret["categories"] = (
                 self._json_data["categories"]
@@ -224,7 +236,18 @@ class _BaseAnnotationWithTool(_BaseAnnotation):
     def type(
         self,
     ) -> Literal["rectangle", "polygon", "semantic", "marker", "vector", "polyline", "pose"]:
-        """Returns the tool of the annotation."""
+        """Returns the tool of the annotation.
+
+        A PDF object detection annotation only carries a "type" key when the tool is not the
+        rectangle: labels produced before the polygon tool existed, and labels produced by the
+        current backend, both omit it for rectangles. The absence of the key is therefore read
+        as "rectangle" instead of raising. This never reaches an export, because serialization
+        reads the raw json data, which this property does not write to.
+        """
+        if "type" not in self._json_data and _is_pdf_object_detection(
+            self._project_info, self._job_interface
+        ):
+            return "rectangle"
         return self._json_data["type"]
 
 
@@ -283,41 +306,81 @@ def bounding_poly(self) -> BoundingPolyList:
         return self._json_data["boundingPoly"]
 
 
-class EntityInPdfAnnotation(_BaseNamedEntityRecognitionAnnotation, _BaseAnnotationWithBoundingPoly):
-    """Class for parsing the "annotations" key of a job response for named entities recognition in PDFs."""
+class _BasePdfAnnotation(_BaseAnnotationWithBoundingPoly):
+    """Base class for the page-level geometry of a PDF annotation.
 
-    @staticmethod
-    def _get_compatible_ml_task() -> Literal["NAMED_ENTITIES_RECOGNITION"]:
-        return "NAMED_ENTITIES_RECOGNITION"
+    In a PDF project the geometry is not on the annotation itself but one level deeper, in a
+    nested "annotations" list, because a PDF page has its own coordinate system. This layout is
+    shared by named entities recognition and object detection.
+
+    The compatibility machinery of `Annotation` matches on the mlTask and the tool, neither of
+    which distinguishes a PDF project from an image one. The input type is therefore checked
+    here, so that an image polygon keeps rejecting `.polys` the way it does today.
+    """
+
+    def _assert_input_type_is_pdf(self, attribute_name: str) -> None:
+        if self._project_info["inputType"] != "PDF":
+            raise AttributeNotCompatibleWithJobError(attribute_name)
 
     @property
     def annotations(self) -> "AnnotationList":
-        """Return the tist of positions of the annotation.
+        """Return the list of positions of the annotation.
 
         For NER, when an annotation spans multiple lines, there will be multiple polys and a single boundingPoly.
         """
-        return AnnotationList(
-            job_name=self._job_name,
-            project_info=self._project_info,
-            annotations_list=self._json_data["annotations"],
-        )
+        self._assert_input_type_is_pdf("annotations")
+        if not isinstance(self._json_data["annotations"], AnnotationList):
+            self._json_data["annotations"] = AnnotationList(
+                job_name=self._job_name,
+                project_info=self._project_info,
+                annotations_list=self._json_data["annotations"],
+            )
+        return self._json_data["annotations"]
 
     @property
     def polys(
         self,
     ) -> list[dict[Literal["normalizedVertices"], list[list[NormalizedVertex]]]]:
-        """Return the coordinates from the different rectangles in the annotation.
+        """Return the coordinates of the shapes in the annotation.
 
-        An annotation can have several rectangles (for example if the annotation covers more than one line).
+        An annotation can have several shapes (for example if a NER annotation covers more than
+        one line). A polygon always has exactly one.
         """
+        self._assert_input_type_is_pdf("polys")
         return self._json_data["polys"]
 
     @property
     def page_number_array(self) -> list[int]:
         """Return the pages where the annotation appears."""
+        self._assert_input_type_is_pdf("page_number_array")
         return self._json_data["pageNumberArray"]
 
 
+class EntityInPdfAnnotation(_BaseNamedEntityRecognitionAnnotation, _BasePdfAnnotation):
+    """Class for parsing the "annotations" key of a job response for named entities recognition in PDFs."""
+
+    @staticmethod
+    def _get_compatible_ml_task() -> Literal["NAMED_ENTITIES_RECOGNITION"]:
+        return "NAMED_ENTITIES_RECOGNITION"
+
+
+class PdfObjectDetectionAnnotation(_BasePdfAnnotation):
+    """Class for parsing the "annotations" key of a job response for object detection in PDFs.
+
+    Registers the page-level geometry attributes for object detection jobs, which is what makes
+    a PDF bounding box or polygon reachable through `.annotations`, `.polys` and
+    `.page_number_array`.
+    """
+
+    @staticmethod
+    def _get_compatible_ml_task() -> Literal["OBJECT_DETECTION"]:
+        return "OBJECT_DETECTION"
+
+    @staticmethod
+    def _get_compatible_type_of_tools() -> Sequence[Literal["rectangle", "polygon", "semantic"]]:
+        return ("rectangle", "polygon", "semantic")
+
+
 class _Base2DAnnotation(_BaseAnnotationWithTool, _BaseAnnotationWithBoundingPoly):
     """Base class for 2D annotations."""
 
@@ -344,7 +407,7 @@ def add_bounding_poly(
                 bounding_poly_list=[],
                 project_info=self._project_info,
                 job_name=self._job_name,
-                type_of_tool=self._json_data["type"],
+                type_of_tool=self._json_data.get("type"),
             )
             if "boundingPoly" not in self._json_data
             else self._json_data["boundingPoly"]
@@ -526,6 +589,7 @@ class Annotation(
     PointAnnotation,
     PolyLineAnnotation,
     EntityInPdfAnnotation,
+    PdfObjectDetectionAnnotation,
     BoundingPolyAnnotation,
     VideoAnnotation,
     PoseEstimationAnnotation,

src/kili/services/label_data_parsing/bounding_poly.py

@@ -57,26 +57,36 @@ def normalized_vertices(
         """Sets the normalized vertices of the bounding polygon.
 
         Args:
-            normalized_vertices: List of normalized vertices for object detection tasks.
-                Or a list of a list of normalized vertices for NER in PDF task.
+            normalized_vertices: A flat list of normalized vertices, or a list of rings
+                (a list of lists of normalized vertices). Both shapes exist in production:
+                image annotations are flat, PDF annotations nest their vertices one level
+                deeper. The nesting is inferred from the data itself, not from the tool,
+                because a PDF annotation is nested whatever tool drew it.
         """
+        rings = (
+            normalized_vertices
+            if normalized_vertices and isinstance(normalized_vertices[0], list)
+            else [normalized_vertices]
+        )
+
         # for object detection tasks type of tool is defined
         if self._type_of_tool:
-            nb_vertices = len(normalized_vertices)
-
-            if self._type_of_tool == "rectangle" and nb_vertices != 4:
-                raise InvalidMutationError(
-                    f"Bounding polygon with {nb_vertices} vertices is not a rectangle."
-                )
-
-            if self._type_of_tool == "polygon" and nb_vertices < 3:
-                raise InvalidMutationError(
-                    f"Bounding polygon with {nb_vertices} vertices is not a polygon."
-                )
-
-        vertices = normalized_vertices if self._type_of_tool else normalized_vertices[0]
-        for vertex in vertices:
-            assert isinstance(vertex, dict), f"Vertex {vertex} is not a dict."
+            for ring in rings:
+                nb_vertices = len(ring)
+
+                if self._type_of_tool == "rectangle" and nb_vertices != 4:
+                    raise InvalidMutationError(
+                        f"Bounding polygon with {nb_vertices} vertices is not a rectangle."
+                    )
+
+                if self._type_of_tool == "polygon" and nb_vertices < 3:
+                    raise InvalidMutationError(
+                        f"Bounding polygon with {nb_vertices} vertices is not a polygon."
+                    )
+
+        for ring in rings:
+            for vertex in ring:
+                assert isinstance(vertex, dict), f"Vertex {vertex} is not a dict."
 
         self._json_data["normalizedVertices"] = normalized_vertices
 

tests/unit/services/export/test_kili.py

@@ -197,6 +197,113 @@ def test_kili_exporter_convert_to_pixel_coords_pdf(mocker: pytest_mock.MockerFix
     }
 
 
+def test_kili_exporter_convert_to_pixel_coords_pdf_polygon(mocker: pytest_mock.MockerFixture):
+    """A PDF polygon scales to pixels against the dimensions of its own page.
+
+    Self-contained on purpose: `fakes/pdf_project_assets.json` is left untouched so that the
+    bounding box expectations keep proving that nothing regressed for existing PDF projects.
+    """
+    mocker.patch.object(KiliExporter, "__init__", return_value=None)
+    exporter = KiliExporter()  # type: ignore  # pylint: disable=no-value-for-parameter
+    exporter.normalized_coordinates = None
+
+    project: ProjectDict = {
+        "id": "fake_project_id",
+        "title": "Fake Project Title",
+        "description": "Fake Project Description",
+        "organizationId": "fake_organization_id",
+        "inputType": "PDF",
+        "jsonInterface": {
+            "jobs": {
+                "OBJECT_DETECTION_JOB": {
+                    "content": {
+                        "categories": {
+                            "A": {"children": [], "color": "#472CED", "name": "A"},
+                            "B": {"children": [], "name": "B", "color": "#5CE7B7"},
+                        },
+                        "input": "radio",
+                    },
+                    "instruction": "Polygon",
+                    "mlTask": "OBJECT_DETECTION",
+                    "required": 1,
+                    "tools": ["polygon"],
+                    "isChild": False,
+                }
+            }
+        },
+    }
+    normalized_vertices = [
+        {"x": 0.1, "y": 0.1},
+        {"x": 0.3, "y": 0.05},
+        {"x": 0.4, "y": 0.3},
+        {"x": 0.25, "y": 0.45},
+        {"x": 0.08, "y": 0.3},
+    ]
+    # page 2 is landscape, so scaling against page 1 would give a different result
+    page_resolutions = [
+        {"pageNumber": 1, "height": 842, "width": 595, "rotation": 0},
+        {"pageNumber": 2, "height": 595, "width": 842, "rotation": 0},
+    ]
+    asset = {
+        "latestLabel": {
+            "author": {
+                "id": "user-feat1-1",
+                "email": "test+admin+1@kili-technology.com",
+                "firstname": "Feat1",
+                "lastname": "Test Admin",
+                "name": "Feat1 Test Admin",
+            },
+            "jsonResponse": {
+                "OBJECT_DETECTION_JOB": {
+                    "annotations": [
+                        {
+                            "children": {},
+                            "annotations": [
+                                {
+                                    "boundingPoly": [{"normalizedVertices": [normalized_vertices]}],
+                                    "pageNumberArray": [2],
+                                    "polys": [{"normalizedVertices": [normalized_vertices]}],
+                                }
+                            ],
+                            "categories": [{"confidence": 100, "name": "A"}],
+                            "content": "",
+                            "mid": "20230703112327217-43948",
+                            "type": "polygon",
+                        },
+                    ]
+                }
+            },
+            "createdAt": "2023-07-03T12:18:08.825Z",
+            "isLatestLabelForUser": True,
+            "labelType": "DEFAULT",
+            "modelName": None,
+        },
+        "pageResolutions": page_resolutions,
+        "content": "https://",
+        "jsonContent": "https://",
+    }
+
+    scaled_asset = convert_to_pixel_coords(asset, project)
+
+    expected_vertices = [
+        [{"x": vertex["x"] * 842, "y": vertex["y"] * 595} for vertex in normalized_vertices]
+    ]
+    scaled_page_annotation = scaled_asset["latestLabel"]["jsonResponse"]["OBJECT_DETECTION_JOB"][
+        "annotations"
+    ][0]["annotations"][0]
+
+    for key in ("polys", "boundingPoly"):
+        assert scaled_page_annotation[key] == [
+            {
+                "normalizedVertices": [normalized_vertices],
+                "vertices": expected_vertices,
+            }
+        ]
+    assert scaled_page_annotation["pageNumberArray"] == [2]
+    # the polygon keeps its five vertices, it is not reduced to a bounding rectangle
+    assert len(scaled_page_annotation["polys"][0]["vertices"][0]) == 5
+
+
 def test_kili_export_labels_non_normalized_pdf(mocker: pytest_mock.MockerFixture):
     get_project_return_val = {
         "inputType": "PDF",