From dff59426213dec330f71e9d3c948f3f63a0bc66b Mon Sep 17 00:00:00 2001 From: Frederic Junod Date: Fri, 28 Aug 2026 17:36:43 +0200 Subject: [PATCH 1/5] fix: read plain tabular JSON and write null-geometry GeoDataFrames _read_file_encoded reads .json files as a table directly via pandas, without touching the .geojson code path. write_data_to_postgis now writes a GeoDataFrame with an entirely-null geometry column as a plain table instead of crashing in to_postgis() ("No valid geometries in the data."). --- .../src/data_manipulation/ingestion.py | 32 ++++++++ .../data_manipulation/tests/test_ingestion.py | 81 +++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/libs/data_manipulation/src/data_manipulation/ingestion.py b/libs/data_manipulation/src/data_manipulation/ingestion.py index 8ce35f81..3dc66edd 100644 --- a/libs/data_manipulation/src/data_manipulation/ingestion.py +++ b/libs/data_manipulation/src/data_manipulation/ingestion.py @@ -1,3 +1,4 @@ +import json import logging import os import re @@ -97,6 +98,19 @@ def _detect_file_encoding(file_path: str) -> str: return encoding or "utf-8" +def _read_tabular_json(file_path: str) -> pd.DataFrame: + """Read a plain (non-GeoJSON) JSON file as a flat table: a list becomes rows, + a single object becomes one row.""" + with open(file_path, "rb") as f: + data = json.load(f) + + if isinstance(data, list): + return pd.DataFrame(data) + if isinstance(data, dict): + return pd.DataFrame([data]) + raise ValueError(f"Unsupported JSON structure in {file_path}: expected an array or object") + + def _read_file_encoded(file_path: str, i: int = 0) -> gpd.GeoDataFrame | pd.DataFrame: """Read a chunk of a geospatial file, handling encoding detection. @@ -123,6 +137,13 @@ def _read_file_encoded(file_path: str, i: int = 0) -> gpd.GeoDataFrame | pd.Data except ValueError: return pd.read_parquet(ds.fragments[i].path) + # Not row-sliceable like a geospatial driver: read fully on the first chunk, + # same pattern as the Parquet branch above. + if Path(file_path).suffix.lower() == ".json": + if i > 0: + return pd.DataFrame() + return _read_tabular_json(file_path) + try: # Try reading with UTF-8 first (common default) result = gpd.read_file(file_path, rows=rows) # type: ignore[arg-type] @@ -728,6 +749,17 @@ def write_data_to_postgis( # Write data to PostGIS as a regular table data.to_sql(table_name, engine, if_exists=if_exists, schema=schema, index=False) + elif data.active_geometry_name is not None and bool( + data[data.active_geometry_name].isna().all() + ): + # to_postgis() can't infer a geometry type with zero non-null geometries + # (e.g. a GeoJSON FeatureCollection with "geometry": null everywhere). + logger.info( + f"Active geometry column '{data.active_geometry_name}' has no non-null " + "geometries; writing as a plain table." + ) + plain_data = pd.DataFrame(data.drop(columns=[data.active_geometry_name])) + plain_data.to_sql(table_name, engine, if_exists=if_exists, schema=schema, index=False) else: # GeoDataFrame # Ensure the geometry column is named 'geom' for PostGIS convention if data.active_geometry_name is None: diff --git a/libs/data_manipulation/tests/test_ingestion.py b/libs/data_manipulation/tests/test_ingestion.py index fb9bfe02..b1e0acba 100644 --- a/libs/data_manipulation/tests/test_ingestion.py +++ b/libs/data_manipulation/tests/test_ingestion.py @@ -1,9 +1,11 @@ """Tests for data ingestion utilities in data_manipulation library.""" +from pathlib import Path from unittest.mock import MagicMock, Mock, patch from urllib.error import URLError import geopandas as gpd +import pandas as pd import pytest import requests from geopandas import GeoDataFrame @@ -65,6 +67,53 @@ def test_parquet_with_geo_metadata_returns_geodataframe(self, mock_read_parquet: assert result is mock_gdf +class TestReadFileEncodedTabularJson: + """.json dispatch in _read_file_encoded (plain tabular JSON, never GeoJSON).""" + + def test_array_of_records(self, tmp_path: Path) -> None: + file_path = tmp_path / "data.json" + file_path.write_text('[{"id": 1, "name": "foo"}, {"id": 2, "name": "bar"}]') + + result = _read_file_encoded(str(file_path)) + + assert result.to_dict("records") == [{"id": 1, "name": "foo"}, {"id": 2, "name": "bar"}] + + def test_single_object_becomes_one_row(self, tmp_path: Path) -> None: + file_path = tmp_path / "data.json" + file_path.write_text('{"id": 1, "name": "foo"}') + + result = _read_file_encoded(str(file_path)) + + assert result.to_dict("records") == [{"id": 1, "name": "foo"}] + + def test_records_with_sparse_keys_union_columns(self, tmp_path: Path) -> None: + file_path = tmp_path / "data.json" + file_path.write_text('[{"id": 1, "name": "foo"}, {"id": 2, "price": 9.99}]') + + result = _read_file_encoded(str(file_path)) + + assert sorted(result.columns) == ["id", "name", "price"] + assert result.loc[0, "name"] == "foo" + assert pd.isna(result.loc[1, "name"]) + assert result.loc[1, "price"] == 9.99 + assert pd.isna(result.loc[0, "price"]) + + def test_unsupported_top_level_scalar_raises(self, tmp_path: Path) -> None: + file_path = tmp_path / "data.json" + file_path.write_text("42") + + with pytest.raises(ValueError, match="Unsupported JSON structure"): + _read_file_encoded(str(file_path)) + + def test_second_chunk_is_empty(self, tmp_path: Path) -> None: + file_path = tmp_path / "data.json" + file_path.write_text('[{"id": 1}, {"id": 2}]') + + result = _read_file_encoded(str(file_path), i=1) + + assert result.empty + + class TestIngestDataFromFileIntoPostgis: """Test cases for ingest_data_from_file_into_postgis function.""" @@ -740,6 +789,38 @@ def test_write_geodataframe_renames_geometry_column( assert gdf.geometry.name == "geom" mock_to_postgis.assert_called_once() + @patch("pandas.DataFrame.to_sql", autospec=True) + @patch("data_manipulation.ingestion._get_table_row_count") + def test_write_geodataframe_with_null_geometry_writes_as_plain_table( + self, + mock_get_row_count: Mock, + mock_to_sql: Mock, + mock_engine: Mock, + ) -> None: + """A geometry column that's entirely null (e.g. a GeoJSON FeatureCollection with + "geometry": null on every feature) is written as a plain table, not to_postgis, + which would otherwise crash trying to infer a PostGIS geometry type.""" + gdf = GeoDataFrame( + {"col1": [1, 2]}, + geometry=gpd.GeoSeries([None, None], name="geometry"), # type: ignore[list-item] + ) + mock_get_row_count.return_value = 2 + + with patch.object(gdf, "to_postgis") as mock_to_postgis: + write_data_to_postgis(gdf, "test_table", mock_engine, "public") + + mock_to_postgis.assert_not_called() + mock_to_sql.assert_called_once() + # autospec=True binds self, so the instance it was called on is args[0]. + written_frame, table_arg = mock_to_sql.call_args.args[0], mock_to_sql.call_args.args[1] + kwargs = mock_to_sql.call_args.kwargs + assert table_arg == "test_table" + assert kwargs["if_exists"] == "replace" + assert kwargs["schema"] == "public" + assert kwargs["index"] is False + # The written frame must not carry the (all-null, useless) geometry column. + assert "geometry" not in written_frame.columns + @patch("data_manipulation.ingestion._get_table_row_count") def test_write_geodataframe_without_geometry( self, From 3ee118af227462df4517de39ae7d933c7652ec09 Mon Sep 17 00:00:00 2001 From: Frederic Junod Date: Fri, 28 Aug 2026 17:36:57 +0200 Subject: [PATCH 2/5] fix: correct json/geojson file_type against the actual staging table The extension-based guess at submission time can be wrong for these two. get_staging_metadata now checks the staging table for a "geom" column (same check read_data_from_postgis already uses) and reports what the data actually turned out to be. Other file types are untouched. --- .../src/api/routes/ingestion/staging.py | 8 +- .../tests/api/routes/test_staging_database.py | 117 +++++++++++++++++- 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/apps/backend/src/api/routes/ingestion/staging.py b/apps/backend/src/api/routes/ingestion/staging.py index 3664eea8..c6e71143 100644 --- a/apps/backend/src/api/routes/ingestion/staging.py +++ b/apps/backend/src/api/routes/ingestion/staging.py @@ -14,7 +14,7 @@ detect_column_type_from_sqla, read_and_transform_data, ) -from data_manipulation.constants import DB_URI_PREFIX +from data_manipulation.constants import DB_URI_PREFIX, DEFAULT_GEOMETRY_COLUMN from data_manipulation.database import schema_exists, table_exists from data_manipulation.ingestion import read_data_from_postgis from data_manipulation.logging import configure_logging @@ -927,6 +927,12 @@ def get_staging_metadata( MetaData(schema=schema), autoload_with=data_engine, ) + if source_file_type in (FileType.JSON, FileType.GEOJSON): + # The extension-based guess made at submission time can be wrong for these + # two: a .geojson source with no real geometry (e.g. an OGC Features + # service that always emits GeoJSON) reads as a plain table, and vice + # versa. Report what the staging table actually turned out to be. + source_file_type = FileType.GEOJSON if DEFAULT_GEOMETRY_COLUMN in table.c else FileType.JSON row_count = data_session.scalar(select(func.count()).select_from(table)) or 0 original_projection = _detect_original_projection( staging_table_name, diff --git a/apps/backend/tests/api/routes/test_staging_database.py b/apps/backend/tests/api/routes/test_staging_database.py index fb19e2c1..bb47151d 100644 --- a/apps/backend/tests/api/routes/test_staging_database.py +++ b/apps/backend/tests/api/routes/test_staging_database.py @@ -1,6 +1,8 @@ """Tests for database source type in staging endpoints.""" +from collections.abc import Generator from datetime import datetime, timezone +from types import SimpleNamespace from unittest.mock import MagicMock, patch from uuid import uuid4 @@ -15,7 +17,7 @@ edit_staging, get_staging_metadata, ) -from src.models.data_import import ImportType +from src.models.data_import import FileType, ImportType, StagingMetadataResponse class TestDbIdentifierValidation: @@ -216,6 +218,36 @@ def test_delete_called_for_file_source(self, mock_delete: MagicMock) -> None: mock_delete.assert_called_once_with("/tmp/somefile.csv") +@pytest.fixture +def staging_metadata_deps() -> Generator[SimpleNamespace, None, None]: + """Patch every dependency get_staging_metadata needs besides the IntegrityLink + and staging table content, which each test configures itself via .load and + .table.""" + with ( + patch("src.api.routes.ingestion.staging.get_staging_schema", return_value="staging"), + patch("src.api.routes.ingestion.staging.select"), + patch("src.api.routes.ingestion.staging.Table") as mock_table, + patch("src.api.routes.ingestion.staging._resolve_columns") as mock_resolve_cols, + patch("src.api.routes.ingestion.staging._detect_original_projection") as mock_detect_proj, + patch("src.api.routes.ingestion.staging.load_authorized_integrity_link") as mock_load, + ): + mock_resolve_cols.return_value = ([], None) + mock_detect_proj.return_value = None + yield SimpleNamespace(load=mock_load, table=mock_table) + + +def _call_get_staging_metadata() -> StagingMetadataResponse: + data_session = MagicMock() + data_session.scalar.return_value = 0 + return get_staging_metadata( + data_session=data_session, + datafeeder_session=MagicMock(), + geo_ctx=MagicMock(), + integrity_link_id=str(uuid4()), + group_ids=[], + ) + + class TestGetStagingMetadataTitleFallback: """Test title fallback logic in get_staging_metadata for database sources.""" @@ -384,6 +416,89 @@ def test_title_preserves_hidden_file_name( assert result.title == ".hidden" +class TestGetStagingMetadataFileTypeOverride: + """The extension-based file_type guess is corrected against the actual staging + table for the two ambiguous cases (json/geojson) since a .geojson source can + turn out to have no real geometry (e.g. an OGC Features service that always + emits GeoJSON), and vice versa.""" + + def test_geojson_without_geom_column_reports_json( + self, staging_metadata_deps: SimpleNamespace + ) -> None: + mock_link = MagicMock() + mock_link.integrity_title = "Tarifs" + mock_link.source_file_name = "data.geojson" + mock_link.source_file_type = FileType.GEOJSON + mock_link.source_import_type = ImportType.URL + mock_link.source_url = None + mock_link.integrity_transformation = None + mock_link.final_table_name = None + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) + staging_metadata_deps.table.return_value.c = {} # no 'geom' column + + result = _call_get_staging_metadata() + + assert result.file_type == FileType.JSON + + def test_geojson_with_geom_column_stays_geojson( + self, staging_metadata_deps: SimpleNamespace + ) -> None: + mock_link = MagicMock() + mock_link.integrity_title = "Parcelles" + mock_link.source_file_name = "data.geojson" + mock_link.source_file_type = FileType.GEOJSON + mock_link.source_import_type = ImportType.URL + mock_link.source_url = None + mock_link.integrity_transformation = None + mock_link.final_table_name = None + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) + staging_metadata_deps.table.return_value.c = {"geom": MagicMock()} + + result = _call_get_staging_metadata() + + assert result.file_type == FileType.GEOJSON + + def test_json_with_geom_column_reports_geojson( + self, staging_metadata_deps: SimpleNamespace + ) -> None: + """Symmetric case: a .json source whose staging table does have a geom + column (e.g. valid GeoJSON uploaded with a .json extension).""" + mock_link = MagicMock() + mock_link.integrity_title = "Points" + mock_link.source_file_name = "data.json" + mock_link.source_file_type = FileType.JSON + mock_link.source_import_type = ImportType.URL + mock_link.source_url = None + mock_link.integrity_transformation = None + mock_link.final_table_name = None + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) + staging_metadata_deps.table.return_value.c = {"geom": MagicMock()} + + result = _call_get_staging_metadata() + + assert result.file_type == FileType.GEOJSON + + def test_non_ambiguous_file_type_is_left_untouched( + self, staging_metadata_deps: SimpleNamespace + ) -> None: + """CSV/GPKG/etc. never go through the json/geojson override, regardless of + whether the staging table happens to have a geom column.""" + mock_link = MagicMock() + mock_link.integrity_title = "Communes" + mock_link.source_file_name = "data.csv" + mock_link.source_file_type = FileType.CSV + mock_link.source_import_type = ImportType.URL + mock_link.source_url = None + mock_link.integrity_transformation = None + mock_link.final_table_name = None + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) + staging_metadata_deps.table.return_value.c = {} + + result = _call_get_staging_metadata() + + assert result.file_type == FileType.CSV + + class TestEditStagingDatabase: """Test edit_staging (PUT) with ImportType.DATABASE.""" From 6f50bf539428d37b68d8b1b673f525c3ce6d109f Mon Sep 17 00:00:00 2001 From: Frederic Junod Date: Fri, 28 Aug 2026 17:37:09 +0200 Subject: [PATCH 3/5] test: dedupe get_staging_metadata patch decorators via shared fixture Moved the repeated six @patch decorators into a staging_metadata_deps fixture in a new conftest.py, shared by test_staging_database.py and test_staging_api.py. --- apps/backend/tests/api/routes/conftest.py | 23 +++ .../tests/api/routes/test_staging_api.py | 73 +++------ .../tests/api/routes/test_staging_database.py | 145 ++---------------- 3 files changed, 59 insertions(+), 182 deletions(-) create mode 100644 apps/backend/tests/api/routes/conftest.py diff --git a/apps/backend/tests/api/routes/conftest.py b/apps/backend/tests/api/routes/conftest.py new file mode 100644 index 00000000..b48530d7 --- /dev/null +++ b/apps/backend/tests/api/routes/conftest.py @@ -0,0 +1,23 @@ +from collections.abc import Generator +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + + +@pytest.fixture +def staging_metadata_deps() -> Generator[SimpleNamespace, None, None]: + """Patch every dependency get_staging_metadata needs besides the IntegrityLink + and staging table content, which each test configures itself via .load and + .table.""" + with ( + patch("src.api.routes.ingestion.staging.get_staging_schema", return_value="staging"), + patch("src.api.routes.ingestion.staging.select"), + patch("src.api.routes.ingestion.staging.Table") as mock_table, + patch("src.api.routes.ingestion.staging._resolve_columns") as mock_resolve_cols, + patch("src.api.routes.ingestion.staging._detect_original_projection") as mock_detect_proj, + patch("src.api.routes.ingestion.staging.load_authorized_integrity_link") as mock_load, + ): + mock_resolve_cols.return_value = ([], None) + mock_detect_proj.return_value = None + yield SimpleNamespace(load=mock_load, table=mock_table) diff --git a/apps/backend/tests/api/routes/test_staging_api.py b/apps/backend/tests/api/routes/test_staging_api.py index 5b306c07..01b97df2 100644 --- a/apps/backend/tests/api/routes/test_staging_api.py +++ b/apps/backend/tests/api/routes/test_staging_api.py @@ -1,6 +1,7 @@ """Tests for API (OGC service) source type in staging endpoints.""" from datetime import date, datetime, timezone +from types import SimpleNamespace from unittest.mock import MagicMock, patch from uuid import uuid4 @@ -16,7 +17,19 @@ edit_staging, get_staging_metadata, ) -from src.models.data_import import ImportType +from src.models.data_import import ImportType, StagingMetadataResponse + + +def _call_get_staging_metadata() -> StagingMetadataResponse: + data_session = MagicMock() + data_session.scalar.return_value = 0 + return get_staging_metadata( + data_session=data_session, + datafeeder_session=MagicMock(), + geo_ctx=MagicMock(), + integrity_link_id=str(uuid4()), + group_ids=[], + ) class TestProcessImportSourceApi: @@ -166,20 +179,8 @@ def test_title_falls_back_to_layer_name( assert result.title == "ns:buildings" - @patch("src.api.routes.ingestion.staging.get_staging_schema", return_value="staging") - @patch("src.api.routes.ingestion.staging.select") - @patch("src.api.routes.ingestion.staging.Table") - @patch("src.api.routes.ingestion.staging._resolve_columns") - @patch("src.api.routes.ingestion.staging._detect_original_projection") - @patch("src.api.routes.ingestion.staging.load_authorized_integrity_link") def test_layer_name_returned_in_response_for_api_type( - self, - mock_load: MagicMock, - mock_detect_proj: MagicMock, - mock_resolve_cols: MagicMock, - mock_table: MagicMock, - mock_select: MagicMock, - mock_get_schema: MagicMock, + self, staging_metadata_deps: SimpleNamespace ) -> None: """layer_name field in response matches source_layer for API import type.""" mock_link = MagicMock() @@ -191,38 +192,15 @@ def test_layer_name_returned_in_response_for_api_type( mock_link.source_layer = "ns:buildings" mock_link.integrity_transformation = None mock_link.final_table_name = None - mock_load.return_value = (mock_link, MagicMock()) - mock_resolve_cols.return_value = ([], None) - mock_detect_proj.return_value = None - - data_session = MagicMock() - data_session.scalar.return_value = 0 + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) - result = get_staging_metadata( - data_session=data_session, - datafeeder_session=MagicMock(), - geo_ctx=MagicMock(), - integrity_link_id=str(uuid4()), - group_ids=[], - ) + result = _call_get_staging_metadata() assert result.title == "My WFS Layer" assert result.layer_name == "ns:buildings" - @patch("src.api.routes.ingestion.staging.get_staging_schema", return_value="staging") - @patch("src.api.routes.ingestion.staging.select") - @patch("src.api.routes.ingestion.staging.Table") - @patch("src.api.routes.ingestion.staging._resolve_columns") - @patch("src.api.routes.ingestion.staging._detect_original_projection") - @patch("src.api.routes.ingestion.staging.load_authorized_integrity_link") def test_layer_name_is_none_for_non_api_import_type( - self, - mock_load: MagicMock, - mock_detect_proj: MagicMock, - mock_resolve_cols: MagicMock, - mock_table: MagicMock, - mock_select: MagicMock, - mock_get_schema: MagicMock, + self, staging_metadata_deps: SimpleNamespace ) -> None: """layer_name is None in response for non-API import types.""" mock_link = MagicMock() @@ -234,20 +212,9 @@ def test_layer_name_is_none_for_non_api_import_type( mock_link.source_layer = None mock_link.integrity_transformation = None mock_link.final_table_name = None - mock_load.return_value = (mock_link, MagicMock()) - mock_resolve_cols.return_value = ([], None) - mock_detect_proj.return_value = None + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) - data_session = MagicMock() - data_session.scalar.return_value = 0 - - result = get_staging_metadata( - data_session=data_session, - datafeeder_session=MagicMock(), - geo_ctx=MagicMock(), - integrity_link_id=str(uuid4()), - group_ids=[], - ) + result = _call_get_staging_metadata() assert result.layer_name is None diff --git a/apps/backend/tests/api/routes/test_staging_database.py b/apps/backend/tests/api/routes/test_staging_database.py index bb47151d..9d3a3276 100644 --- a/apps/backend/tests/api/routes/test_staging_database.py +++ b/apps/backend/tests/api/routes/test_staging_database.py @@ -1,6 +1,5 @@ """Tests for database source type in staging endpoints.""" -from collections.abc import Generator from datetime import datetime, timezone from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -218,24 +217,6 @@ def test_delete_called_for_file_source(self, mock_delete: MagicMock) -> None: mock_delete.assert_called_once_with("/tmp/somefile.csv") -@pytest.fixture -def staging_metadata_deps() -> Generator[SimpleNamespace, None, None]: - """Patch every dependency get_staging_metadata needs besides the IntegrityLink - and staging table content, which each test configures itself via .load and - .table.""" - with ( - patch("src.api.routes.ingestion.staging.get_staging_schema", return_value="staging"), - patch("src.api.routes.ingestion.staging.select"), - patch("src.api.routes.ingestion.staging.Table") as mock_table, - patch("src.api.routes.ingestion.staging._resolve_columns") as mock_resolve_cols, - patch("src.api.routes.ingestion.staging._detect_original_projection") as mock_detect_proj, - patch("src.api.routes.ingestion.staging.load_authorized_integrity_link") as mock_load, - ): - mock_resolve_cols.return_value = ([], None) - mock_detect_proj.return_value = None - yield SimpleNamespace(load=mock_load, table=mock_table) - - def _call_get_staging_metadata() -> StagingMetadataResponse: data_session = MagicMock() data_session.scalar.return_value = 0 @@ -251,20 +232,8 @@ def _call_get_staging_metadata() -> StagingMetadataResponse: class TestGetStagingMetadataTitleFallback: """Test title fallback logic in get_staging_metadata for database sources.""" - @patch("src.api.routes.ingestion.staging.get_staging_schema", return_value="staging") - @patch("src.api.routes.ingestion.staging.select") - @patch("src.api.routes.ingestion.staging.Table") - @patch("src.api.routes.ingestion.staging._resolve_columns") - @patch("src.api.routes.ingestion.staging._detect_original_projection") - @patch("src.api.routes.ingestion.staging.load_authorized_integrity_link") def test_title_falls_back_to_table_name_from_source_url( - self, - mock_load: MagicMock, - mock_detect_proj: MagicMock, - mock_resolve_cols: MagicMock, - mock_table: MagicMock, - mock_select: MagicMock, - mock_get_schema: MagicMock, + self, staging_metadata_deps: SimpleNamespace ) -> None: """Title is the table name parsed from db://{schema}/{table} when no custom title is set.""" mock_link = MagicMock() @@ -275,37 +244,14 @@ def test_title_falls_back_to_table_name_from_source_url( mock_link.source_url = "db://SOURCE_DB_1/geo/parcels" mock_link.integrity_transformation = None mock_link.final_table_name = None - mock_load.return_value = (mock_link, MagicMock()) - mock_resolve_cols.return_value = ([], None) - mock_detect_proj.return_value = None - - data_session = MagicMock() - data_session.scalar.return_value = 0 - - result = get_staging_metadata( - data_session=data_session, - datafeeder_session=MagicMock(), - geo_ctx=MagicMock(), - integrity_link_id=str(uuid4()), - group_ids=[], - ) + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) + + result = _call_get_staging_metadata() assert result.title == "parcels" - @patch("src.api.routes.ingestion.staging.get_staging_schema", return_value="staging") - @patch("src.api.routes.ingestion.staging.select") - @patch("src.api.routes.ingestion.staging.Table") - @patch("src.api.routes.ingestion.staging._resolve_columns") - @patch("src.api.routes.ingestion.staging._detect_original_projection") - @patch("src.api.routes.ingestion.staging.load_authorized_integrity_link") def test_custom_title_overrides_table_name( - self, - mock_load: MagicMock, - mock_detect_proj: MagicMock, - mock_resolve_cols: MagicMock, - mock_table: MagicMock, - mock_select: MagicMock, - mock_get_schema: MagicMock, + self, staging_metadata_deps: SimpleNamespace ) -> None: """integrity_title takes precedence over table name derived from source_url.""" mock_link = MagicMock() @@ -316,37 +262,14 @@ def test_custom_title_overrides_table_name( mock_link.source_url = "db://SOURCE_DB_1/geo/parcels" mock_link.integrity_transformation = None mock_link.final_table_name = None - mock_load.return_value = (mock_link, MagicMock()) - mock_resolve_cols.return_value = ([], None) - mock_detect_proj.return_value = None - - data_session = MagicMock() - data_session.scalar.return_value = 0 - - result = get_staging_metadata( - data_session=data_session, - datafeeder_session=MagicMock(), - geo_ctx=MagicMock(), - integrity_link_id=str(uuid4()), - group_ids=[], - ) + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) + + result = _call_get_staging_metadata() assert result.title == "Parcelles cadastrales" - @patch("src.api.routes.ingestion.staging.get_staging_schema", return_value="staging") - @patch("src.api.routes.ingestion.staging.select") - @patch("src.api.routes.ingestion.staging.Table") - @patch("src.api.routes.ingestion.staging._resolve_columns") - @patch("src.api.routes.ingestion.staging._detect_original_projection") - @patch("src.api.routes.ingestion.staging.load_authorized_integrity_link") def test_title_strips_extension_from_source_file_name( - self, - mock_load: MagicMock, - mock_detect_proj: MagicMock, - mock_resolve_cols: MagicMock, - mock_table: MagicMock, - mock_select: MagicMock, - mock_get_schema: MagicMock, + self, staging_metadata_deps: SimpleNamespace ) -> None: """source_file_name extension is stripped when used as title fallback.""" mock_link = MagicMock() @@ -357,38 +280,13 @@ def test_title_strips_extension_from_source_file_name( mock_link.source_url = None mock_link.integrity_transformation = None mock_link.final_table_name = None - mock_load.return_value = (mock_link, MagicMock()) - mock_resolve_cols.return_value = ([], None) - mock_detect_proj.return_value = None - - data_session = MagicMock() - data_session.scalar.return_value = 0 - - result = get_staging_metadata( - data_session=data_session, - datafeeder_session=MagicMock(), - geo_ctx=MagicMock(), - integrity_link_id=str(uuid4()), - group_ids=[], - ) + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) + + result = _call_get_staging_metadata() assert result.title == "data" - @patch("src.api.routes.ingestion.staging.get_staging_schema", return_value="staging") - @patch("src.api.routes.ingestion.staging.select") - @patch("src.api.routes.ingestion.staging.Table") - @patch("src.api.routes.ingestion.staging._resolve_columns") - @patch("src.api.routes.ingestion.staging._detect_original_projection") - @patch("src.api.routes.ingestion.staging.load_authorized_integrity_link") - def test_title_preserves_hidden_file_name( - self, - mock_load: MagicMock, - mock_detect_proj: MagicMock, - mock_resolve_cols: MagicMock, - mock_table: MagicMock, - mock_select: MagicMock, - mock_get_schema: MagicMock, - ) -> None: + def test_title_preserves_hidden_file_name(self, staging_metadata_deps: SimpleNamespace) -> None: """Hidden-file names (leading dot) are kept as-is without stripping the extension.""" mock_link = MagicMock() mock_link.integrity_title = None @@ -398,20 +296,9 @@ def test_title_preserves_hidden_file_name( mock_link.source_url = None mock_link.integrity_transformation = None mock_link.final_table_name = None - mock_load.return_value = (mock_link, MagicMock()) - mock_resolve_cols.return_value = ([], None) - mock_detect_proj.return_value = None - - data_session = MagicMock() - data_session.scalar.return_value = 0 - - result = get_staging_metadata( - data_session=data_session, - datafeeder_session=MagicMock(), - geo_ctx=MagicMock(), - integrity_link_id=str(uuid4()), - group_ids=[], - ) + staging_metadata_deps.load.return_value = (mock_link, MagicMock()) + + result = _call_get_staging_metadata() assert result.title == ".hidden" From 2fde32c20290d55c968c6b4a6ca79cf9bdea7c25 Mon Sep 17 00:00:00 2001 From: Frederic Junod Date: Mon, 31 Aug 2026 10:04:52 +0200 Subject: [PATCH 4/5] fix: reject .geojson sources whose geometry is null across the whole file Checks every feature's geometry once, via a separate geometry-only read of the whole file, so chunking doesn't affect the verdict. OGC WFS/OAPIF services keep their own null-geometry handling, unaffected. --- .../src/data_manipulation/ingestion.py | 24 +++- .../data_manipulation/tests/test_ingestion.py | 110 +++++++++++++++++- 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/libs/data_manipulation/src/data_manipulation/ingestion.py b/libs/data_manipulation/src/data_manipulation/ingestion.py index 3dc66edd..3f07915d 100644 --- a/libs/data_manipulation/src/data_manipulation/ingestion.py +++ b/libs/data_manipulation/src/data_manipulation/ingestion.py @@ -111,6 +111,23 @@ def _read_tabular_json(file_path: str) -> pd.DataFrame: raise ValueError(f"Unsupported JSON structure in {file_path}: expected an array or object") +def _reject_geojson_without_geometry(file_path: str) -> None: + """Raise if a .geojson source has a null geometry on every feature. Checked + against the whole file (one geometry-only read), not just the chunk being + returned, so chunking doesn't affect the verdict. OGC WFS/OAPIF services are + exempt — they have their own null-geometry handling in + ingest_data_from_ogc_service_into_postgis. + """ + if Path(file_path).suffix.lower() != ".geojson": + return + geometries = gpd.read_file(file_path, columns=[]) # type: ignore[arg-type] + if not geometries.empty and bool(geometries.geometry.isna().all()): + raise ValueError( + f"GeoJSON file {file_path} has no valid geometries: every feature's geometry " + "is null. Use a .json extension if this data is meant to be tabular." + ) + + def _read_file_encoded(file_path: str, i: int = 0) -> gpd.GeoDataFrame | pd.DataFrame: """Read a chunk of a geospatial file, handling encoding detection. @@ -154,6 +171,8 @@ def _read_file_encoded(file_path: str, i: int = 0) -> gpd.GeoDataFrame | pd.Data for column in result.columns: if pd.api.types.is_string_dtype(result[column].dtype): result[column].to_numpy() + if i == 0: + _reject_geojson_without_geometry(file_path) return result except (UnicodeDecodeError, ArrowException): logger.warning( @@ -163,7 +182,10 @@ def _read_file_encoded(file_path: str, i: int = 0) -> gpd.GeoDataFrame | pd.Data # Detect encoding (mainly for shapefiles, others default to UTF-8) encoding = _detect_file_encoding(file_path) logger.warning("Detected encoding: %s", encoding) - return gpd.read_file(file_path, rows=rows, encoding=encoding) # type: ignore[arg-type] + result = gpd.read_file(file_path, rows=rows, encoding=encoding) # type: ignore[arg-type] + if i == 0: + _reject_geojson_without_geometry(file_path) + return result def ingest_data_from_file_into_postgis( diff --git a/libs/data_manipulation/tests/test_ingestion.py b/libs/data_manipulation/tests/test_ingestion.py index b1e0acba..26be453b 100644 --- a/libs/data_manipulation/tests/test_ingestion.py +++ b/libs/data_manipulation/tests/test_ingestion.py @@ -114,6 +114,76 @@ def test_second_chunk_is_empty(self, tmp_path: Path) -> None: assert result.empty +class TestReadFileEncodedGeojsonNullGeometry: + """.geojson sources whose geometry is null on every feature must raise, not + silently fall back to a plain table (OGC WFS/OAPIF services keep that fallback, + handled elsewhere).""" + + @patch("data_manipulation.ingestion.gpd.read_file") + def test_geojson_all_null_geometry_raises(self, mock_read_file: Mock) -> None: + mock_read_file.return_value = GeoDataFrame({"col1": [1, 2], "geometry": [None, None]}) + + with pytest.raises(ValueError, match="no valid geometries"): + _read_file_encoded("data.geojson") + + @patch("data_manipulation.ingestion.gpd.read_file") + def test_geojson_with_some_geometry_does_not_raise(self, mock_read_file: Mock) -> None: + mock_gdf = GeoDataFrame({"col1": [1, 2], "geometry": [Point(0, 0), None]}) + mock_read_file.return_value = mock_gdf + + result = _read_file_encoded("data.geojson") + + assert result is mock_gdf + + @patch("data_manipulation.ingestion.gpd.read_file") + def test_non_geojson_all_null_geometry_does_not_raise(self, mock_read_file: Mock) -> None: + """Shapefile/GeoPackage/etc. keep the generic null-geometry fallback.""" + mock_gdf = GeoDataFrame({"col1": [1, 2], "geometry": [None, None]}) + mock_read_file.return_value = mock_gdf + + result = _read_file_encoded("data.gpkg") + + assert result is mock_gdf + + @patch("data_manipulation.ingestion.gpd.read_file") + def test_geojson_empty_terminating_chunk_does_not_raise(self, mock_read_file: Mock) -> None: + """End-of-file signal (empty chunk) isn't treated as all-null geometry.""" + mock_read_file.return_value = GeoDataFrame({"col1": [], "geometry": []}) + + result = _read_file_encoded("data.geojson", i=1) + + assert result.empty + + @patch("data_manipulation.ingestion.gpd.read_file") + def test_geojson_all_null_trailing_chunk_does_not_raise(self, mock_read_file: Mock) -> None: + """A null-geometry chunk beyond the first doesn't abort the import.""" + mock_gdf = GeoDataFrame({"col1": [1, 2], "geometry": [None, None]}) + mock_read_file.return_value = mock_gdf + + result = _read_file_encoded("data.geojson", i=1) + + assert result is mock_gdf + + @patch("data_manipulation.ingestion.gpd.read_file") + def test_geojson_geometry_only_in_later_chunk_does_not_raise( + self, mock_read_file: Mock + ) -> None: + """The whole-file check must see geometry even if chunk 0 itself is null.""" + + def read_file_side_effect( + file_path: str, rows: object = None, columns: object = None, **kwargs: object + ) -> GeoDataFrame: + if columns == []: + return GeoDataFrame({"geometry": [None, Point(0, 0)]}) + return GeoDataFrame({"col1": [1], "geometry": [None]}) + + mock_read_file.side_effect = read_file_side_effect + + result = _read_file_encoded("data.geojson", i=0) + + assert result.to_dict("records") == [{"col1": 1, "geometry": None}] + + class TestIngestDataFromFileIntoPostgis: """Test cases for ingest_data_from_file_into_postgis function.""" @@ -153,7 +223,10 @@ def test_read_file_encoded_utf8_success(self, mock_read_file: Mock) -> None: result = _read_file_encoded("test.geojson") - mock_read_file.assert_called_once_with("test.geojson", rows=slice(0, CHUNK_SIZE, None)) + # .geojson also triggers the whole-file null-geometry check + mock_read_file.assert_any_call("test.geojson", rows=slice(0, CHUNK_SIZE, None)) + mock_read_file.assert_any_call("test.geojson", columns=[]) + assert mock_read_file.call_count == 2 assert result is mock_gdf @patch("data_manipulation.ingestion.gpd.read_file") @@ -217,7 +290,8 @@ def test_ingest_from_url_success( mock_requests_get.assert_called_once_with( "http://example.com/data.geojson", auth=None, timeout=300 ) - mock_read_file.assert_called_once() + # .geojson also triggers the whole-file null-geometry check + assert mock_read_file.call_count == 2 mock_write_data.assert_called_once_with( mock_gdf, "test_table", mock_engine, "public", if_exists="replace" ) @@ -282,11 +356,41 @@ def test_ingest_from_url_with_content_disposition( ) mock_requests_get.assert_called_once() - mock_read_file.assert_called_once() + # .geojson also triggers the whole-file null-geometry check + assert mock_read_file.call_count == 2 # Verify that the file was saved with the extracted filename call_args = mock_read_file.call_args[0][0] assert "data.geojson" in str(call_args) + @patch("data_manipulation.ingestion.write_data_to_postgis") + @patch("data_manipulation.ingestion.gpd.read_file") + @patch("data_manipulation.ingestion.requests.get") + def test_ingest_from_url_geojson_all_null_geometry_raises( + self, + mock_requests_get: Mock, + mock_read_file: Mock, + mock_write_data: Mock, + mock_engine: Mock, + ) -> None: + """A .geojson URL source with no valid geometry fails ingestion outright + instead of being silently written as a non-spatial table.""" + + mock_response = Mock() + mock_response.status_code = 200 + mock_response.content = b"test data" + mock_response.headers = {} + mock_requests_get.return_value = mock_response + + mock_gdf = GeoDataFrame({"col1": [1, 2], "geometry": [None, None]}) + mock_read_file.return_value = mock_gdf + + with pytest.raises(ValueError, match="no valid geometries"): + ingest_data_from_url_into_postgis( + "http://example.com/data.geojson", "test_table", mock_engine, "public" + ) + + mock_write_data.assert_not_called() + @patch("data_manipulation.ingestion.requests.get") def test_ingest_from_url_http_error( self, From 6795d429a058c0f8c1e9c705735cd2d76542329b Mon Sep 17 00:00:00 2001 From: Frederic Junod Date: Tue, 1 Sep 2026 07:56:53 +0200 Subject: [PATCH 5/5] feat: run a jq filter on plain tabular JSON sources Adds the jq PyPI package (bundles libjq, no system dependency) and runs raw JSON text through it before parsing, seeded with the fixed identity filter "." for now. This is the seam a future editable filter plugs into. Frontend shows a disabled "JQ filter" field prefilled with "." next to the dataset title, for json sources only. No editing or persistence yet. --- apps/elt/uv.lock | 17 +++++++ .../data-import-wizard.component.html | 2 + .../data-import-wizard.component.ts | 2 + .../dataset-jq-filter.component.html | 10 ++++ .../dataset-jq-filter.component.spec.ts | 48 +++++++++++++++++++ .../dataset-jq-filter.component.ts | 18 +++++++ apps/frontend/translations/de.json | 3 +- apps/frontend/translations/en.json | 1 + apps/frontend/translations/es.json | 3 +- apps/frontend/translations/fr.json | 1 + apps/frontend/translations/it.json | 3 +- apps/frontend/translations/nl.json | 3 +- apps/frontend/translations/pt.json | 3 +- apps/frontend/translations/sk.json | 3 +- libs/data_manipulation/pyproject.toml | 1 + .../src/data_manipulation/ingestion.py | 15 ++++-- .../data_manipulation/tests/test_ingestion.py | 15 ++++++ uv.lock | 17 +++++++ 18 files changed, 156 insertions(+), 9 deletions(-) create mode 100644 apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.html create mode 100644 apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.spec.ts create mode 100644 apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.ts diff --git a/apps/elt/uv.lock b/apps/elt/uv.lock index ffda84d8..d34d4dc0 100644 --- a/apps/elt/uv.lock +++ b/apps/elt/uv.lock @@ -608,6 +608,7 @@ dependencies = [ { name = "geoalchemy2" }, { name = "geopandas" }, { name = "geoservercloud" }, + { name = "jq" }, { name = "pyarrow" }, { name = "pydantic" }, { name = "sqlalchemy" }, @@ -627,6 +628,7 @@ requires-dist = [ { name = "geoalchemy2", specifier = "==0.19.0" }, { name = "geopandas", specifier = "==1.1.3" }, { name = "geoservercloud", git = "https://github.com/camptocamp/python-geoservercloud.git" }, + { name = "jq", specifier = ">=1.12.0" }, { name = "pyarrow", specifier = "==24.0.0" }, { name = "pydantic", specifier = "==2.13.4" }, { name = "sqlalchemy", specifier = "==2.0.49" }, @@ -1133,6 +1135,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] +[[package]] +name = "jq" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/ec/3da01457bbd3c6a2fc8fea6736c0b657ffc628e3decbfb1fafcf33dc7dbe/jq-1.12.0.tar.gz", hash = "sha256:729b2d3418c8ca7dccfaa66b9fb7a98bec28474212650d27c5c04358ce26f55c", size = 1965902, upload-time = "2026-07-10T12:36:39.214Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/4b/4c7510b06e2f1b59cb83b9cef71a756e4ae7cc868309be3a55a43fb64ffa/jq-1.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4df3d3feb9d9f2526416261c56a46808c07987283dbb3d272f0941839b9154a7", size = 416281, upload-time = "2026-07-10T12:35:21.807Z" }, + { url = "https://files.pythonhosted.org/packages/38/10/19f037f1ff857e0398c0548803ce09dc9dd962978b83d7afeb25a6008986/jq-1.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39cb15398a13c604051848c60598f1367f7dbb34c336ec495a5bd804429c9801", size = 424919, upload-time = "2026-07-10T12:35:23.475Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f3/3982e172926699eb6c5d30d7004ee6cf922837c16adba9327d3321e88255/jq-1.12.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:860d71c12ae8ce3bd5210193e7ca2acf19d521bb0c2d8fddd3e7571fa66c4ada", size = 757286, upload-time = "2026-07-10T12:35:24.868Z" }, + { url = "https://files.pythonhosted.org/packages/ec/50/90f9c6f54a5e87e388b1837171be7c6ddd9d02be6eed4df5b2281d6f83ee/jq-1.12.0-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59aeba4ce1a5b1a012025277d8266c769044cb33f0b1a1b6354bf5ee2987c408", size = 776143, upload-time = "2026-07-10T12:35:27.051Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e9/9e53b968add6455cd19acb5a01abdba6f06ee6dd8c3c45dd907ad1da7542/jq-1.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73d91b05f83ad61e9af9b09588fb959742ca096d4c696b2685fd9daa716ed0b0", size = 744439, upload-time = "2026-07-10T12:35:28.893Z" }, + { url = "https://files.pythonhosted.org/packages/45/51/75fb4883151152cc09a98091f2c4b0ec1e39686ab90a4a7453d0531bfb22/jq-1.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:46cbcb69732dcbe582b8af2bf24f8982dd063c21319285db38c1a1948c540dac", size = 770644, upload-time = "2026-07-10T12:35:30.534Z" }, + { url = "https://files.pythonhosted.org/packages/05/c1/026afa899f3f5b475a2f715640c70f54323c8ac5f1964bff1e6d5f75cd8b/jq-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0920d06c7f115fc7ce461f246a58a2dd7594ac491f214974ab3e5b367ea70da", size = 412137, upload-time = "2026-07-10T12:35:32.069Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" diff --git a/apps/frontend/src/app/shared/components/data-import-wizard/data-import-wizard.component.html b/apps/frontend/src/app/shared/components/data-import-wizard/data-import-wizard.component.html index 8f4f815f..638401d4 100644 --- a/apps/frontend/src/app/shared/components/data-import-wizard/data-import-wizard.component.html +++ b/apps/frontend/src/app/shared/components/data-import-wizard/data-import-wizard.component.html @@ -63,6 +63,8 @@

/> + + diff --git a/apps/frontend/src/app/shared/components/data-import-wizard/data-import-wizard.component.ts b/apps/frontend/src/app/shared/components/data-import-wizard/data-import-wizard.component.ts index dcd545e5..438c8600 100644 --- a/apps/frontend/src/app/shared/components/data-import-wizard/data-import-wizard.component.ts +++ b/apps/frontend/src/app/shared/components/data-import-wizard/data-import-wizard.component.ts @@ -59,6 +59,7 @@ import type { SourceData } from '../data-source-selector/data-source-selector.co import { DataSourceSelectorComponent } from '../data-source-selector/data-source-selector.component' import { SettingsService } from '../../../core/settings/settings.service' import { DatasetTitleComponent } from '../dataset-title/dataset-title.component' +import { DatasetJqFilterComponent } from '../dataset-jq-filter/dataset-jq-filter.component' import { DatasetConfigurationComponent } from '../dataset-configuration/dataset-configuration.component' import type { ColumnAction, @@ -109,6 +110,7 @@ export interface ImportWizardData { SpinningLoaderComponent, DataSourceSelectorComponent, DatasetTitleComponent, + DatasetJqFilterComponent, DatasetConfigurationComponent, TranslatePipe, DatasetPreviewTableComponent, diff --git a/apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.html b/apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.html new file mode 100644 index 00000000..669e7f88 --- /dev/null +++ b/apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.html @@ -0,0 +1,10 @@ +@if (metadata()?.file_type === 'json') { +
+

+ {{ 'import.configuration.jqFilter' | translate }} +

+ + + +
+} diff --git a/apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.spec.ts b/apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.spec.ts new file mode 100644 index 00000000..1d87635f --- /dev/null +++ b/apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.spec.ts @@ -0,0 +1,48 @@ +import { ComponentRef } from '@angular/core' +import { TestBed } from '@angular/core/testing' +import { DatasetJqFilterComponent } from './dataset-jq-filter.component' +import { TranslateTestingModule } from 'ngx-translate-testing' +import type { StagingMetadataResponse } from '../../../core/api/models' + +const buildMetadata = ( + overrides: Partial = {} +): StagingMetadataResponse => + ({ + file_type: 'json', + ...overrides + } as StagingMetadataResponse) + +describe('DatasetJqFilterComponent', () => { + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [ + DatasetJqFilterComponent, + TranslateTestingModule.withTranslations({ + en: { 'import.configuration.jqFilter': 'JQ filter' } + }).withDefaultLanguage('en') + ] + }).compileComponents() + }) + + it('shows the disabled jq filter field prefilled with "." for a json source', () => { + const fixture = TestBed.createComponent(DatasetJqFilterComponent) + const ref = fixture.componentRef as ComponentRef + ref.setInput('metadata', buildMetadata({ file_type: 'json' })) + fixture.detectChanges() + + const textarea: HTMLTextAreaElement = + fixture.nativeElement.querySelector('textarea') + expect(textarea).toBeTruthy() + expect(textarea.value).toBe('.') + expect(textarea.disabled).toBe(true) + }) + + it('does not render for a non-json source', () => { + const fixture = TestBed.createComponent(DatasetJqFilterComponent) + const ref = fixture.componentRef as ComponentRef + ref.setInput('metadata', buildMetadata({ file_type: 'geojson' })) + fixture.detectChanges() + + expect(fixture.nativeElement.querySelector('textarea')).toBeFalsy() + }) +}) diff --git a/apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.ts b/apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.ts new file mode 100644 index 00000000..c5209c3f --- /dev/null +++ b/apps/frontend/src/app/shared/components/dataset-jq-filter/dataset-jq-filter.component.ts @@ -0,0 +1,18 @@ +import { ChangeDetectionStrategy, Component, input } from '@angular/core' +import { MatFormFieldModule } from '@angular/material/form-field' +import { MatInputModule } from '@angular/material/input' +import { TranslatePipe } from '@ngx-translate/core' +import type { StagingMetadataResponse } from '../../../core/api/models' + +@Component({ + selector: 'app-dataset-jq-filter', + changeDetection: ChangeDetectionStrategy.OnPush, + imports: [MatFormFieldModule, MatInputModule, TranslatePipe], + templateUrl: './dataset-jq-filter.component.html' +}) +export class DatasetJqFilterComponent { + metadata = input(null) + + // Not yet user-configurable; the seam a future editable jq filter plugs into. + readonly defaultFilter = '.' +} diff --git a/apps/frontend/translations/de.json b/apps/frontend/translations/de.json index 5ab5f8c4..f45b7e96 100644 --- a/apps/frontend/translations/de.json +++ b/apps/frontend/translations/de.json @@ -217,6 +217,7 @@ "import.columnHeader.error.empty": "Der Spaltenname darf nicht leer sein", "import.configuration.datasetTitle": "", "import.configuration.datasetTitlePlaceholder": "", + "import.configuration.jqFilter": "", "import.configuration.loading": "", "import.configuration.maxLengthTitle": "", "import.configuration.minLengthTitle": "", @@ -337,4 +338,4 @@ "status.type.running": "", "status.type.success": "", "status.type.warning": "" -} \ No newline at end of file +} diff --git a/apps/frontend/translations/en.json b/apps/frontend/translations/en.json index bc26c223..9cfd642e 100644 --- a/apps/frontend/translations/en.json +++ b/apps/frontend/translations/en.json @@ -217,6 +217,7 @@ "import.columnHeader.error.empty": "Column name cannot be empty", "import.configuration.datasetTitle": "Data title", "import.configuration.datasetTitlePlaceholder": "Enter a title for your dataset", + "import.configuration.jqFilter": "JQ filter", "import.configuration.loading": "Data loading...", "import.configuration.maxLengthTitle": "255 characters maximum", "import.configuration.minLengthTitle": "3 characters minimum", diff --git a/apps/frontend/translations/es.json b/apps/frontend/translations/es.json index 1de66bdb..b8af9326 100644 --- a/apps/frontend/translations/es.json +++ b/apps/frontend/translations/es.json @@ -217,6 +217,7 @@ "import.columnHeader.error.empty": "", "import.configuration.datasetTitle": "", "import.configuration.datasetTitlePlaceholder": "", + "import.configuration.jqFilter": "", "import.configuration.loading": "", "import.configuration.maxLengthTitle": "", "import.configuration.minLengthTitle": "", @@ -337,4 +338,4 @@ "status.type.running": "", "status.type.success": "", "status.type.warning": "" -} \ No newline at end of file +} diff --git a/apps/frontend/translations/fr.json b/apps/frontend/translations/fr.json index ba685306..900b588a 100644 --- a/apps/frontend/translations/fr.json +++ b/apps/frontend/translations/fr.json @@ -218,6 +218,7 @@ "import.columnHeader.error.empty": "Le nom de la colonne ne peut pas être vide.", "import.configuration.datasetTitle": "Titre du jeu de données", "import.configuration.datasetTitlePlaceholder": "Entrez un titre pour votre jeu de données", + "import.configuration.jqFilter": "Filtre JQ", "import.configuration.loading": "Chargement des données...", "import.configuration.maxLengthTitle": "255 caractères maximum", "import.configuration.minLengthTitle": "3 caractères minimum", diff --git a/apps/frontend/translations/it.json b/apps/frontend/translations/it.json index e8ff6889..78f959f7 100644 --- a/apps/frontend/translations/it.json +++ b/apps/frontend/translations/it.json @@ -217,6 +217,7 @@ "import.columnHeader.error.empty": "", "import.configuration.datasetTitle": "", "import.configuration.datasetTitlePlaceholder": "", + "import.configuration.jqFilter": "", "import.configuration.loading": "", "import.configuration.maxLengthTitle": "", "import.configuration.minLengthTitle": "", @@ -337,4 +338,4 @@ "status.type.running": "", "status.type.success": "", "status.type.warning": "" -} \ No newline at end of file +} diff --git a/apps/frontend/translations/nl.json b/apps/frontend/translations/nl.json index e2361273..287e3ae8 100644 --- a/apps/frontend/translations/nl.json +++ b/apps/frontend/translations/nl.json @@ -217,6 +217,7 @@ "import.columnHeader.error.empty": "", "import.configuration.datasetTitle": "", "import.configuration.datasetTitlePlaceholder": "", + "import.configuration.jqFilter": "", "import.configuration.loading": "", "import.configuration.maxLengthTitle": "", "import.configuration.minLengthTitle": "", @@ -337,4 +338,4 @@ "status.type.running": "", "status.type.success": "", "status.type.warning": "" -} \ No newline at end of file +} diff --git a/apps/frontend/translations/pt.json b/apps/frontend/translations/pt.json index 5ad8e608..4f5d7cf4 100644 --- a/apps/frontend/translations/pt.json +++ b/apps/frontend/translations/pt.json @@ -217,6 +217,7 @@ "import.columnHeader.error.empty": "", "import.configuration.datasetTitle": "", "import.configuration.datasetTitlePlaceholder": "", + "import.configuration.jqFilter": "", "import.configuration.loading": "", "import.configuration.maxLengthTitle": "", "import.configuration.minLengthTitle": "", @@ -337,4 +338,4 @@ "status.type.running": "", "status.type.success": "", "status.type.warning": "" -} \ No newline at end of file +} diff --git a/apps/frontend/translations/sk.json b/apps/frontend/translations/sk.json index d52859cb..97f4c5b7 100644 --- a/apps/frontend/translations/sk.json +++ b/apps/frontend/translations/sk.json @@ -217,6 +217,7 @@ "import.columnHeader.error.empty": "", "import.configuration.datasetTitle": "", "import.configuration.datasetTitlePlaceholder": "", + "import.configuration.jqFilter": "", "import.configuration.loading": "", "import.configuration.maxLengthTitle": "", "import.configuration.minLengthTitle": "", @@ -337,4 +338,4 @@ "status.type.running": "", "status.type.success": "", "status.type.warning": "" -} \ No newline at end of file +} diff --git a/libs/data_manipulation/pyproject.toml b/libs/data_manipulation/pyproject.toml index 5a127a29..e028e1d0 100644 --- a/libs/data_manipulation/pyproject.toml +++ b/libs/data_manipulation/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "geoalchemy2==0.19.0", "geopandas==1.1.3", "geoservercloud", + "jq>=1.12.0", "pyarrow==24.0.0", "pydantic==2.13.4", "sqlalchemy==2.0.49", diff --git a/libs/data_manipulation/src/data_manipulation/ingestion.py b/libs/data_manipulation/src/data_manipulation/ingestion.py index 3f07915d..9b84cd18 100644 --- a/libs/data_manipulation/src/data_manipulation/ingestion.py +++ b/libs/data_manipulation/src/data_manipulation/ingestion.py @@ -1,4 +1,3 @@ -import json import logging import os import re @@ -13,6 +12,7 @@ import chardet import geopandas as gpd +import jq import pandas as pd import pyarrow.parquet as pq import requests @@ -98,11 +98,20 @@ def _detect_file_encoding(file_path: str) -> str: return encoding or "utf-8" +# Not yet user-configurable; the seam a future editable jq filter plugs into. +DEFAULT_JQ_FILTER = "." + + +def _apply_jq_filter(text: str, filter_expr: str = DEFAULT_JQ_FILTER) -> object: + """Run a jq filter against raw JSON text, returning the filtered Python value.""" + return jq.compile(filter_expr).input_text(text).first() + + def _read_tabular_json(file_path: str) -> pd.DataFrame: """Read a plain (non-GeoJSON) JSON file as a flat table: a list becomes rows, a single object becomes one row.""" - with open(file_path, "rb") as f: - data = json.load(f) + with open(file_path, encoding="utf-8") as f: + data = _apply_jq_filter(f.read()) if isinstance(data, list): return pd.DataFrame(data) diff --git a/libs/data_manipulation/tests/test_ingestion.py b/libs/data_manipulation/tests/test_ingestion.py index 26be453b..c244dd7b 100644 --- a/libs/data_manipulation/tests/test_ingestion.py +++ b/libs/data_manipulation/tests/test_ingestion.py @@ -17,6 +17,7 @@ from data_manipulation.constants import POSTGIS_TABLE_NAME_MAX_LENGTH from data_manipulation.ingestion import ( CHUNK_SIZE, + _apply_jq_filter, # pyright: ignore[reportPrivateUsage] _read_file_encoded, # pyright: ignore[reportPrivateUsage] ingest_data_from_database_into_postgis, ingest_data_from_file_into_postgis, @@ -67,6 +68,20 @@ def test_parquet_with_geo_metadata_returns_geodataframe(self, mock_read_parquet: assert result is mock_gdf +class TestApplyJqFilter: + """_apply_jq_filter runs the (currently fixed) default jq filter.""" + + def test_identity_filter_returns_list_unchanged(self) -> None: + result = _apply_jq_filter('[{"id": 1, "name": "foo"}, {"id": 2, "name": "bar"}]') + + assert result == [{"id": 1, "name": "foo"}, {"id": 2, "name": "bar"}] + + def test_identity_filter_returns_object_unchanged(self) -> None: + result = _apply_jq_filter('{"id": 1, "name": "foo"}') + + assert result == {"id": 1, "name": "foo"} + + class TestReadFileEncodedTabularJson: """.json dispatch in _read_file_encoded (plain tabular JSON, never GeoJSON).""" diff --git a/uv.lock b/uv.lock index 9d049083..40737445 100644 --- a/uv.lock +++ b/uv.lock @@ -279,6 +279,7 @@ dependencies = [ { name = "geoalchemy2" }, { name = "geopandas" }, { name = "geoservercloud" }, + { name = "jq" }, { name = "pyarrow" }, { name = "pydantic" }, { name = "sqlalchemy" }, @@ -298,6 +299,7 @@ requires-dist = [ { name = "geoalchemy2", specifier = "==0.19.0" }, { name = "geopandas", specifier = "==1.1.3" }, { name = "geoservercloud", git = "https://github.com/camptocamp/python-geoservercloud.git" }, + { name = "jq", specifier = ">=1.12.0" }, { name = "pyarrow", specifier = "==24.0.0" }, { name = "pydantic", specifier = "==2.13.4" }, { name = "sqlalchemy", specifier = "==2.0.49" }, @@ -717,6 +719,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/6f/55bc5837e9fe7a86a5acb553adec901257d709062bfaef7debd4d8cfee12/jproperties-2.1.2-py2.py3-none-any.whl", hash = "sha256:4108e868353a9f4a12bb86a92df5462d0e18d00119169533972ce473029be79a", size = 17981, upload-time = "2024-07-21T20:40:49.034Z" }, ] +[[package]] +name = "jq" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/ec/3da01457bbd3c6a2fc8fea6736c0b657ffc628e3decbfb1fafcf33dc7dbe/jq-1.12.0.tar.gz", hash = "sha256:729b2d3418c8ca7dccfaa66b9fb7a98bec28474212650d27c5c04358ce26f55c", size = 1965902, upload-time = "2026-07-10T12:36:39.214Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/4b/4c7510b06e2f1b59cb83b9cef71a756e4ae7cc868309be3a55a43fb64ffa/jq-1.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4df3d3feb9d9f2526416261c56a46808c07987283dbb3d272f0941839b9154a7", size = 416281, upload-time = "2026-07-10T12:35:21.807Z" }, + { url = "https://files.pythonhosted.org/packages/38/10/19f037f1ff857e0398c0548803ce09dc9dd962978b83d7afeb25a6008986/jq-1.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39cb15398a13c604051848c60598f1367f7dbb34c336ec495a5bd804429c9801", size = 424919, upload-time = "2026-07-10T12:35:23.475Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f3/3982e172926699eb6c5d30d7004ee6cf922837c16adba9327d3321e88255/jq-1.12.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:860d71c12ae8ce3bd5210193e7ca2acf19d521bb0c2d8fddd3e7571fa66c4ada", size = 757286, upload-time = "2026-07-10T12:35:24.868Z" }, + { url = "https://files.pythonhosted.org/packages/ec/50/90f9c6f54a5e87e388b1837171be7c6ddd9d02be6eed4df5b2281d6f83ee/jq-1.12.0-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59aeba4ce1a5b1a012025277d8266c769044cb33f0b1a1b6354bf5ee2987c408", size = 776143, upload-time = "2026-07-10T12:35:27.051Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e9/9e53b968add6455cd19acb5a01abdba6f06ee6dd8c3c45dd907ad1da7542/jq-1.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73d91b05f83ad61e9af9b09588fb959742ca096d4c696b2685fd9daa716ed0b0", size = 744439, upload-time = "2026-07-10T12:35:28.893Z" }, + { url = "https://files.pythonhosted.org/packages/45/51/75fb4883151152cc09a98091f2c4b0ec1e39686ab90a4a7453d0531bfb22/jq-1.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:46cbcb69732dcbe582b8af2bf24f8982dd063c21319285db38c1a1948c540dac", size = 770644, upload-time = "2026-07-10T12:35:30.534Z" }, + { url = "https://files.pythonhosted.org/packages/05/c1/026afa899f3f5b475a2f715640c70f54323c8ac5f1964bff1e6d5f75cd8b/jq-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0920d06c7f115fc7ce461f246a58a2dd7594ac491f214974ab3e5b367ea70da", size = 412137, upload-time = "2026-07-10T12:35:32.069Z" }, +] + [[package]] name = "jsonschema" version = "4.25.1"