diff --git a/CHANGELOG.md b/CHANGELOG.md index 746d9ec0..ed9c5716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,8 +55,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/). toolchain, for production deployments where the Rust build environment is not needed. - Dependabot coverage expanded to Docker base images and pip packages (two new ecosystems with grouped update policies). - -### Changed - `pypgstac migrate` now delegates runtime migration planning and apply logic to `pgstac-migrate`; `src/pypgstac/src/pypgstac/migrate.py` remains as a compatibility wrapper. @@ -111,6 +109,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). `python-runtime` groups). - `docker-compose.yml` removes explicit `container_name` entries to avoid conflicts between concurrent local instances. +- Appended `collection` as a secondary sorting tie-breaker before `id` inside `keyset_sortkeys`. This ensures deterministic pagination tokens globally and enables PostgreSQL partition pruning for optimized multi-collection searches. +- Updated pgTAP test suite expectations and output snapshots to match the new `collection, id` sorting structure. ### Removed - PL/Rust support: `pgstacbase-plrust` and `pgstac-plrust` Docker targets removed; the @@ -135,6 +135,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - `scripts/container-scripts/test` now derives the active database from `PGDATABASE`/`POSTGRES_DB` when checking server extensions and refreshing collation versions, instead of assuming `postgis`. +- Fixed an edge-case bug where keyset pagination returned 0 features on Page 2 if items across different collections shared identical IDs and timestamps (#392). +- Resolved a Python type-checking diagnostic in `pypgstac.load` regarding dynamic file opening modes. ## [v0.9.11] diff --git a/src/pgstac/pgstac.sql b/src/pgstac/pgstac.sql index 0383cb94..83f2b510 100644 --- a/src/pgstac/pgstac.sql +++ b/src/pgstac/pgstac.sql @@ -2293,7 +2293,7 @@ BEGIN END IF; IF jsonb_typeof(j) = 'object' THEN - -- GeoJSON geometry args (Point/Polygon/.../GeometryCollection) are not cql1 expressions; + -- GeoJSON geometry args (Point/Polygon/.../GeometryCollection) are not cql2 expressions; -- pass them through unchanged so spatial ops keep their geometry intact. IF j ? 'type' AND (j ? 'coordinates' OR j ? 'geometries') THEN RETURN j; @@ -2870,9 +2870,10 @@ RETURNS TABLE(ord int, field text, expr text, dir text, notnull boolean) AS $$ SELECT coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]'::jsonb) AS s ), app AS ( + -- Append collection first (helps partition pruning), then id to guarantee global uniqueness SELECT s - || jsonb_build_object('field','id','direction', s->0->>'direction') - || jsonb_build_object('field','collection','direction', s->0->>'direction') AS s + || jsonb_build_object('field','collection','direction', s->0->>'direction') + || jsonb_build_object('field','id','direction', s->0->>'direction') AS s FROM base ), rows AS ( @@ -5829,7 +5830,6 @@ BEGIN END LOOP; END; $$ LANGUAGE PLPGSQL; - -- END FRAGMENT: 997_maintenance.sql -- BEGIN FRAGMENT: 998_idempotent_post.sql diff --git a/src/pgstac/sql/002d_keyset.sql b/src/pgstac/sql/002d_keyset.sql index 8c4fb399..c7f3afde 100644 --- a/src/pgstac/sql/002d_keyset.sql +++ b/src/pgstac/sql/002d_keyset.sql @@ -26,9 +26,10 @@ RETURNS TABLE(ord int, field text, expr text, dir text, notnull boolean) AS $$ SELECT coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]'::jsonb) AS s ), app AS ( + -- Append collection first (helps partition pruning), then id to guarantee global uniqueness SELECT s - || jsonb_build_object('field','id','direction', s->0->>'direction') - || jsonb_build_object('field','collection','direction', s->0->>'direction') AS s + || jsonb_build_object('field','collection','direction', s->0->>'direction') + || jsonb_build_object('field','id','direction', s->0->>'direction') AS s FROM base ), rows AS ( diff --git a/src/pgstac/tests/basic/collection_searches.sql.out b/src/pgstac/tests/basic/collection_searches.sql.out index 80807665..fa1d98af 100644 --- a/src/pgstac/tests/basic/collection_searches.sql.out +++ b/src/pgstac/tests/basic/collection_searches.sql.out @@ -75,10 +75,11 @@ SELECT page, ids FROM pg_temp.collwalk('{"ids":["testcollection_1","testcollecti 2 | ["testcollection_1"] select collection_search('{"filter":{"op":"eq", "args":[{"property":"title"},"My Test Collection."]},"limit":10, "sortby":[{"field":"datetime","direction":"desc"}]}'); - {"links": [{"rel": "next", "href": "./collections?token=next:MjAxMi0wMy0zMSAwMDowMDowMCswMB90ZXN0Y29sbGVjdGlvbl82MzkfY29sbGVjdGlvbnM=", "type": "application/json", "method": "GET"}], "collections": [{"id": "testcollection_648", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[170, 80, 180, 90]]}, "temporal": {"interval": [["2012-06-02 00:00:00+00", "2012-08-02 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_647", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[160, 80, 170, 90]]}, "temporal": {"interval": [["2012-05-26 00:00:00+00", "2012-07-26 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_646", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[150, 80, 160, 90]]}, "temporal": {"interval": [["2012-05-19 00:00:00+00", "2012-07-19 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_645", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[140, 80, 150, 90]]}, "temporal": {"interval": [["2012-05-12 00:00:00+00", "2012-07-12 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_644", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[130, 80, 140, 90]]}, "temporal": {"interval": [["2012-05-05 00:00:00+00", "2012-07-05 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_643", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[120, 80, 130, 90]]}, "temporal": {"interval": [["2012-04-28 00:00:00+00", "2012-06-28 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_642", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[110, 80, 120, 90]]}, "temporal": {"interval": [["2012-04-21 00:00:00+00", "2012-06-21 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_641", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[100, 80, 110, 90]]}, "temporal": {"interval": [["2012-04-14 00:00:00+00", "2012-06-14 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_640", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[90, 80, 100, 90]]}, "temporal": {"interval": [["2012-04-07 00:00:00+00", "2012-06-07 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_639", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[80, 80, 90, 90]]}, "temporal": {"interval": [["2012-03-31 00:00:00+00", "2012-05-31 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}], "numberMatched": 648, "numberReturned": 10} + {"links": [{"rel": "next", "href": "./collections?token=next:MjAxMi0wMy0zMSAwMDowMDowMCswMB9jb2xsZWN0aW9ucx90ZXN0Y29sbGVjdGlvbl82Mzk=", "type": "application/json", "method": "GET"}], "collections": [{"id": "testcollection_648", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[170, 80, 180, 90]]}, "temporal": {"interval": [["2012-06-02 00:00:00+00", "2012-08-02 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_647", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[160, 80, 170, 90]]}, "temporal": {"interval": [["2012-05-26 00:00:00+00", "2012-07-26 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_646", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[150, 80, 160, 90]]}, "temporal": {"interval": [["2012-05-19 00:00:00+00", "2012-07-19 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_645", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[140, 80, 150, 90]]}, "temporal": {"interval": [["2012-05-12 00:00:00+00", "2012-07-12 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_644", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[130, 80, 140, 90]]}, "temporal": {"interval": [["2012-05-05 00:00:00+00", "2012-07-05 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_643", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[120, 80, 130, 90]]}, "temporal": {"interval": [["2012-04-28 00:00:00+00", "2012-06-28 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_642", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[110, 80, 120, 90]]}, "temporal": {"interval": [["2012-04-21 00:00:00+00", "2012-06-21 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_641", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[100, 80, 110, 90]]}, "temporal": {"interval": [["2012-04-14 00:00:00+00", "2012-06-14 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_640", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[90, 80, 100, 90]]}, "temporal": {"interval": [["2012-04-07 00:00:00+00", "2012-06-07 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_639", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[80, 80, 90, 90]]}, "temporal": {"interval": [["2012-03-31 00:00:00+00", "2012-05-31 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}], "numberMatched": 648, "numberReturned": 10} select collection_search('{"datetime":["2012-01-01","2012-01-02"], "filter":{"op":"eq", "args":[{"property":"title"},"My Test Collection."]},"limit":10, "sortby":[{"field":"datetime","direction":"desc"}]}'); {"links": [], "collections": [{"id": "testcollection_626", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-50, 80, -40, 90]]}, "temporal": {"interval": [["2011-12-31 00:00:00+00", "2012-02-29 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_625", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-60, 80, -50, 90]]}, "temporal": {"interval": [["2011-12-24 00:00:00+00", "2012-02-24 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_624", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-70, 80, -60, 90]]}, "temporal": {"interval": [["2011-12-17 00:00:00+00", "2012-02-17 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_623", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-80, 80, -70, 90]]}, "temporal": {"interval": [["2011-12-10 00:00:00+00", "2012-02-10 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_622", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-90, 80, -80, 90]]}, "temporal": {"interval": [["2011-12-03 00:00:00+00", "2012-02-03 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_621", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-100, 80, -90, 90]]}, "temporal": {"interval": [["2011-11-26 00:00:00+00", "2012-01-26 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_620", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-110, 80, -100, 90]]}, "temporal": {"interval": [["2011-11-19 00:00:00+00", "2012-01-19 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_619", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-120, 80, -110, 90]]}, "temporal": {"interval": [["2011-11-12 00:00:00+00", "2012-01-12 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_618", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-130, 80, -120, 90]]}, "temporal": {"interval": [["2011-11-05 00:00:00+00", "2012-01-05 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}], "numberMatched": 9, "numberReturned": 9} select collection_search('{"ids":["testcollection_1","testcollection_2"], "fields": {"include": ["title"]}}'); {"links": [], "collections": [{"id": "testcollection_1", "title": "My Test Collection."}, {"id": "testcollection_2", "title": "My Test Collection."}], "numberMatched": 2, "numberReturned": 2} + diff --git a/src/pgstac/tests/basic/cql2_searches.sql.out b/src/pgstac/tests/basic/cql2_searches.sql.out index 38b98fb0..ae50c7ca 100644 --- a/src/pgstac/tests/basic/cql2_searches.sql.out +++ b/src/pgstac/tests/basic/cql2_searches.sql.out @@ -37,7 +37,7 @@ SELECT search('{"ids":["pgstac-test-item-0097","pgstac-test-item-0003"],"fields" {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 2, "numberMatched" : 2} SELECT search('{"collections":["pgstac-test-collection"],"fields":{"include":["id"]}, "limit": 1}'); - {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1pdGVtLTAwMDMfcGdzdGFjLXRlc3Qt\nY29sbGVjdGlvbg==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 1, "numberMatched" : 100} + {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1jb2xsZWN0aW9uH3Bnc3RhYy10ZXN0\nLWl0ZW0tMDAwMw==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 1, "numberMatched" : 100} SELECT search('{"collections":["something"]}'); {"type" : "FeatureCollection", "features" : [], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 0, "numberMatched" : 0} @@ -58,7 +58,7 @@ SELECT search('{"filter":{"op":"eq", "args":[{"property":"eo:cloud_cover"},36]}, {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0087", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 36}}, {"id": "pgstac-test-item-0089", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 36}}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 2, "numberMatched" : 2} SELECT search('{"filter":{"op":"lt", "args":[{"property":"eo:cloud_cover"},25]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"eo:cloud_cover","direction":"asc"}]}'); - {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 1}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0063", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0034", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0041", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0073", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0048", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 4}}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:NB9wZ3N0YWMtdGVzdC1pdGVtLTAwNDgfcGdzdGFjLXRlc3QtY29sbGVjdGlvbg==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 10, "numberMatched" : 31} + {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 1}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0063", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0034", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0041", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0073", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0048", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 4}}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:NB9wZ3N0YWMtdGVzdC1jb2xsZWN0aW9uH3Bnc3RhYy10ZXN0LWl0ZW0tMDA0OA==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 10, "numberMatched" : 31} SELECT search('{"filter":{"op":"in","args":[{"property":"id"},["pgstac-test-item-0097"]]},"fields":{"include":["id"]}}'); {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 1, "numberMatched" : 1} @@ -67,7 +67,7 @@ SELECT search('{"filter":{"op":"in","args":[{"property":"id"},["pgstac-test-item {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 2, "numberMatched" : 2} SELECT search('{"filter":{"op":"in","args":[{"property":"collection"},["pgstac-test-collection"]]},"fields":{"include":["id"]}, "limit": 1}'); - {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1pdGVtLTAwMDMfcGdzdGFjLXRlc3Qt\nY29sbGVjdGlvbg==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 1, "numberMatched" : 100} + {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1jb2xsZWN0aW9uH3Bnc3RhYy10ZXN0\nLWl0ZW0tMDAwMw==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 1, "numberMatched" : 100} SELECT search('{"filter":{"op":"in","args":[{"property":"collection"},["nonexistent"]]}}'); {"type" : "FeatureCollection", "features" : [], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 0, "numberMatched" : 0} @@ -76,7 +76,7 @@ SELECT search('{"filter":{"op":"in","args":[{"property":"collection"},["nonexist {"type" : "FeatureCollection", "features" : [], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 0} SELECT search('{"conf": {"nohydrate": true}, "limit": 2}'); - {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}, {"id": "pgstac-test-item-0002", "bbox": [-85.504167, 30.934008, -85.433293, 31.003486], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008505_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.434414, 30.934008], [-85.433293, 31.002658], [-85.503096, 31.003486], [-85.504167, 30.934834], [-85.434414, 30.934008]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [642906, 3423339, 649572, 3430950], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7611, 6666], "eo:cloud_cover": 33, "proj:transform": [1, 0, 642906, 0, -1, 3430950, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1pdGVtLTAwMDIfcGdzdGFjLXRlc3Qt\nY29sbGVjdGlvbg==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 2, "numberMatched" : 100} + {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}, {"id": "pgstac-test-item-0002", "bbox": [-85.504167, 30.934008, -85.433293, 31.003486], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008505_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.434414, 30.934008], [-85.433293, 31.002658], [-85.503096, 31.003486], [-85.504167, 30.934834], [-85.434414, 30.934008]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [642906, 3423339, 649572, 3430950], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7611, 6666], "eo:cloud_cover": 33, "proj:transform": [1, 0, 642906, 0, -1, 3430950, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1jb2xsZWN0aW9uH3Bnc3RhYy10ZXN0\nLWl0ZW0tMDAwMg==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 2, "numberMatched" : 100} SELECT search('{"filter":{"op":"in","args":[{"property":"naip:state"},["zz","xx"]]},"fields":{"include":["id"]}}'); {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0100", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 2, "numberMatched" : 2} @@ -131,10 +131,11 @@ SELECT page, ids FROM pg_temp.pagewalk('{"limit":3,"sortby":[{"field":"propertie 4 | ["pgstac-test-item-0048", "pgstac-test-item-0054", "pgstac-test-item-0032"] SELECT search('{"collections": ["pgstac-test-collection"], "limit": 1}'); - {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1pdGVtLTAwMDMfcGdzdGFjLXRlc3Qt\nY29sbGVjdGlvbg==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 1, "numberMatched" : 100} + {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1jb2xsZWN0aW9uH3Bnc3RhYy10ZXN0\nLWl0ZW0tMDAwMw==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 1, "numberMatched" : 100} -- limit 1 paging: walk forward following the real keyset next-token. SELECT page, ids FROM pg_temp.pagewalk('{"collections":["pgstac-test-collection"],"limit":1}'::jsonb, 3); 1 | ["pgstac-test-item-0003"] 2 | ["pgstac-test-item-0002"] 3 | ["pgstac-test-item-0001"] + diff --git a/src/pgstac/tests/basic/cql_searches.sql.out b/src/pgstac/tests/basic/cql_searches.sql.out index c60a5458..bc0c0627 100644 --- a/src/pgstac/tests/basic/cql_searches.sql.out +++ b/src/pgstac/tests/basic/cql_searches.sql.out @@ -55,7 +55,7 @@ SELECT search('{"filter":{"eq":[{"property":"eo:cloud_cover"},36]}, "fields":{"i {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0087", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 36}}, {"id": "pgstac-test-item-0089", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 36}}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 2, "numberMatched" : 2} SELECT search('{"filter":{"lt":[{"property":"eo:cloud_cover"},25]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"eo:cloud_cover","direction":"asc"}]}'); - {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 1}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0063", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0034", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0041", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0073", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0048", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 4}}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:NB9wZ3N0YWMtdGVzdC1pdGVtLTAwNDgfcGdzdGFjLXRlc3QtY29sbGVjdGlvbg==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 10, "numberMatched" : 31} + {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 1}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0063", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0034", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0041", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0073", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0048", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 4}}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:NB9wZ3N0YWMtdGVzdC1jb2xsZWN0aW9uH3Bnc3RhYy10ZXN0LWl0ZW0tMDA0OA==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 10, "numberMatched" : 31} SELECT search('{"ids":["pgstac-test-item-0097"],"fields":{"include":["id"]}}'); {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 1, "numberMatched" : 1} @@ -67,7 +67,7 @@ SELECT search('{"ids":["bogusid"],"fields":{"include":["id"]}}'); {"type" : "FeatureCollection", "features" : [], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 0, "numberMatched" : 0} SELECT search('{"collections":["pgstac-test-collection"],"fields":{"include":["id"]}, "limit": 1}'); - {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1pdGVtLTAwMDMfcGdzdGFjLXRlc3Qt\nY29sbGVjdGlvbg==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 1, "numberMatched" : 100} + {"type" : "FeatureCollection", "features" : [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:MjAxMS0wOC0yNSAwMDowMDowMCswMB9wZ3N0YWMtdGVzdC1jb2xsZWN0aW9uH3Bnc3RhYy10ZXN0\nLWl0ZW0tMDAwMw==", "type": "application/geo+json", "method": "GET"}], "numberReturned" : 1, "numberMatched" : 100} SELECT search('{"collections":["something"]}'); {"type" : "FeatureCollection", "features" : [], "links" : [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "numberReturned" : 0, "numberMatched" : 0} @@ -80,7 +80,8 @@ SELECT s.usecount IS NOT NULL and s.usecount > 0 AND s.lastused IS NOT NULL AND SELECT s.hash, s.search, s._where, s.orderby, s.metadata from search_query('{"collections":["pgstac-test-collection"]}'::jsonb, _metadata=>'{"meta":"value"}'::jsonb) q JOIN searches s ON s.hash = q.hash; SELECT s.hash, s.search, s._where, s.orderby, s.metadata from search_query('{"collections":["pgstac-test-collection"]}'::jsonb, _metadata=>'{"meta":"value"}'::jsonb) q JOIN searches s ON s.hash = q.hash; - 3213c65cda868bbfd3443717a82961f42f7987af1af54f0e2639d17a7bb29391 | {"collections": ["pgstac-test-collection"]} | collection IN ('pgstac-test-collection') | datetime DESC, id DESC, collection DESC | {"meta": "value"} + 3213c65cda868bbfd3443717a82961f42f7987af1af54f0e2639d17a7bb29391 | {"collections": ["pgstac-test-collection"]} | collection IN ('pgstac-test-collection') | datetime DESC, collection DESC, id DESC | {"meta": "value"} SELECT s.usecount IS NOT NULL and s.usecount > 0 AND s.lastused IS NOT NULL AND s.lastused < clock_timestamp() FROM search_query('{"collections":["pgstac-test-collection"]}') q JOIN searches s ON s.hash = q.hash; t + diff --git a/src/pgstac/tests/basic/crud_functions.sql.out b/src/pgstac/tests/basic/crud_functions.sql.out index 0e4de1b6..44d1e1c9 100644 --- a/src/pgstac/tests/basic/crud_functions.sql.out +++ b/src/pgstac/tests/basic/crud_functions.sql.out @@ -25,7 +25,7 @@ INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-cr INSERT 0 1 -- Create an item SELECT create_item((SELECT content FROM test_items LIMIT 1)); - + SELECT id, geometry, collection, datetime, end_datetime, properties, extra FROM items WHERE collection='pgstactest-crudtest' ORDER BY id; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2020-01-01 00:00:00+00 | 2020-01-01 00:00:00+00 | {} | {} @@ -36,7 +36,7 @@ SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; -- Update item with new datetime that is in a different partition SELECT update_item((SELECT content || '{"properties":{"datetime":"2023-01-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); - + SELECT id, geometry, collection, datetime, end_datetime, properties, extra FROM items WHERE collection='pgstactest-crudtest' ORDER BY id; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2023-01-01 00:00:00+00 | 2023-01-01 00:00:00+00 | {} | {} @@ -47,21 +47,21 @@ SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; -- Update item with new datetime that is in a different partition SELECT upsert_item((SELECT content || '{"properties":{"datetime":"2023-02-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); - + SELECT id, geometry, collection, datetime, end_datetime, properties, extra FROM items WHERE collection='pgstactest-crudtest' ORDER BY id; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2023-02-01 00:00:00+00 | 2023-02-01 00:00:00+00 | {} | {} -- Delete an item SELECT delete_item('pgstactest-crudtest-1', 'pgstactest-crudtest'); - + SELECT id, geometry, collection, datetime, end_datetime, properties, extra FROM items WHERE collection='pgstactest-crudtest' ORDER BY id; WITH c AS (SELECT content FROM test_items LIMIT 2), aggregated AS (SELECT jsonb_agg(content) as items FROM c) SELECT create_items(items) FROM aggregated; - + SELECT id, geometry, collection, datetime, end_datetime, properties, extra FROM items WHERE collection='pgstactest-crudtest' ORDER BY id; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2020-01-01 00:00:00+00 | 2020-01-01 00:00:00+00 | {} | {} @@ -73,7 +73,7 @@ DELETE 2 WITH c AS (SELECT content FROM test_items LIMIT 2), aggregated AS (SELECT jsonb_agg(content) as items FROM c) SELECT upsert_items(items) FROM aggregated; - + SELECT id, geometry, collection, datetime, end_datetime, properties, extra FROM items WHERE collection='pgstactest-crudtest' ORDER BY id; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2020-01-01 00:00:00+00 | 2020-01-01 00:00:00+00 | {} | {} @@ -83,7 +83,7 @@ SELECT id, geometry, collection, datetime, end_datetime, properties, extra FROM WITH c AS (SELECT content || '{"properties":{"datetime":"2023-02-01 00:00:00Z"}}'::jsonb as content FROM test_items LIMIT 2), aggregated AS (SELECT jsonb_agg(content) as items FROM c) SELECT upsert_items(items) FROM aggregated; - + SELECT id, geometry, collection, datetime, end_datetime, properties, extra FROM items WHERE collection='pgstactest-crudtest' ORDER BY id; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2023-02-01 00:00:00+00 | 2023-02-01 00:00:00+00 | {} | {} @@ -96,7 +96,7 @@ SELECT get_setting_bool('update_collection_extent'); f SELECT update_item((SELECT content || '{"properties":{"datetime":"2024-01-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); - + SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2023-02-01T00:00:00+00:00", "2023-02-01T00:00:00+00:00"]]}} diff --git a/src/pgstac/tests/basic/hydration.sql.out b/src/pgstac/tests/basic/hydration.sql.out index 374453a1..1bb638c1 100644 --- a/src/pgstac/tests/basic/hydration.sql.out +++ b/src/pgstac/tests/basic/hydration.sql.out @@ -15,7 +15,7 @@ SELECT create_item('{ "end_datetime": "2026-01-31T23:00:00Z" } }'); - + -- Verify datetime is stripped from stored properties (promoted temporal field) SELECT properties ? 'datetime' FROM items WHERE id='temporal-range-item'; diff --git a/src/pgstac/tests/pgtap.sql b/src/pgstac/tests/pgtap.sql index 78e1d8d4..987c774b 100644 --- a/src/pgstac/tests/pgtap.sql +++ b/src/pgstac/tests/pgtap.sql @@ -17,7 +17,7 @@ CREATE EXTENSION IF NOT EXISTS pgtap; SET SEARCH_PATH TO pgstac, pgtap, public; -- Plan the tests. -SELECT plan(357); +SELECT plan(363); --SELECT * FROM no_plan(); -- Run the tests. diff --git a/src/pgstac/tests/pgtap/004_search.sql b/src/pgstac/tests/pgtap/004_search.sql index 2dda96a4..d192234b 100644 --- a/src/pgstac/tests/pgtap/004_search.sql +++ b/src/pgstac/tests/pgtap/004_search.sql @@ -41,7 +41,7 @@ SELECT has_function('pgstac'::name, 'keyset_orderby', ARRAY['jsonb','boolean']); SELECT results_eq($$ SELECT keyset_orderby('{"sortby":[{"field":"datetime","direction":"desc"},{"field":"eo:cloud_cover","direction":"asc"}]}'::jsonb); $$,$$ - SELECT 'datetime DESC, eo_cloud_cover ASC, id DESC, collection DESC'; + SELECT 'datetime DESC, eo_cloud_cover ASC, collection DESC, id DESC'; $$, 'Test creation of sort sql' ); @@ -50,7 +50,7 @@ SELECT results_eq($$ SELECT results_eq($$ SELECT keyset_orderby('{"sortby":[{"field":"datetime","direction":"desc"},{"field":"eo:cloud_cover","direction":"asc"}]}'::jsonb, true); $$,$$ - SELECT 'datetime ASC, eo_cloud_cover DESC, id ASC, collection ASC'; + SELECT 'datetime ASC, eo_cloud_cover DESC, collection ASC, id ASC'; $$, 'Test creation of reverse sort sql' ); @@ -833,3 +833,74 @@ SELECT is( '2', 'Make sure all matching items are returned when items with the same ID are in multiple collections, all collections specified. #192' ); + +-- Keyset pagination across a duplicate id in two collections with identical datetime (issue #392). +-- The collection tie-breaker must let page 2's look-ahead filter step past the shared (datetime, id) +-- instead of collapsing to zero features. +SELECT is( + (search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb)->>'numberReturned')::int, + 1, + 'Duplicate id pagination: page 1 returns exactly one item. #392' +); + +SELECT is( + ( + WITH p1 AS (SELECT search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb)::jsonb AS j), + nt AS (SELECT split_part(jsonb_path_query_first((SELECT j FROM p1), '$.links[*] ? (@.rel == "next").href')->>0, 'token=', 2) AS t) + SELECT (search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb || jsonb_build_object('token', (SELECT t FROM nt)))->>'numberReturned')::int + ), + 1, + 'Duplicate id pagination: page 2 steps past the tie-breaker and returns the second item. #392' +); + +-- Page 1 and page 2 must surface the item from two different collections (both duplicates covered). +SELECT is( + ( + WITH p1 AS (SELECT search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb)::jsonb AS j), + nt AS (SELECT split_part(jsonb_path_query_first((SELECT j FROM p1), '$.links[*] ? (@.rel == "next").href')->>0, 'token=', 2) AS t), + p2 AS (SELECT search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb || jsonb_build_object('token', (SELECT t FROM nt)))::jsonb AS j) + SELECT count(DISTINCT c)::int FROM ( + SELECT jsonb_path_query_first((SELECT j FROM p1), '$.features[0].collection')->>0 AS c + UNION ALL + SELECT jsonb_path_query_first((SELECT j FROM p2), '$.features[0].collection')->>0 AS c + ) s + ), + 2, + 'Duplicate id pagination: the two pages cover both collections. #392' +); + +-- Page 2 must expose a valid prev link back to page 1. +SELECT isnt( + ( + WITH p1 AS (SELECT search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb)::jsonb AS j), + nt AS (SELECT split_part(jsonb_path_query_first((SELECT j FROM p1), '$.links[*] ? (@.rel == "next").href')->>0, 'token=', 2) AS t), + p2 AS (SELECT search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb || jsonb_build_object('token', (SELECT t FROM nt)))::jsonb AS j) + SELECT jsonb_path_query_first((SELECT j FROM p2), '$.links[*] ? (@.rel == "prev").href')->>0 + ), + NULL, + 'Duplicate id pagination: page 2 generates a valid prev token. #392' +); + +-- Reverse pagination: traveling backward via prev token successfully loads 1 item. +SELECT is( + ( + WITH p1 AS (SELECT search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb)::jsonb AS j), + nt AS (SELECT split_part(jsonb_path_query_first((SELECT j FROM p1), '$.links[*] ? (@.rel == "next").href')->>0, 'token=', 2) AS t), + p2 AS (SELECT search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb || jsonb_build_object('token', (SELECT t FROM nt)))::jsonb AS j), + pt AS (SELECT split_part(jsonb_path_query_first((SELECT j FROM p2), '$.links[*] ? (@.rel == "prev").href')->>0, 'token=', 2) AS t) + SELECT (search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1}'::jsonb || jsonb_build_object('token', (SELECT t FROM pt)))->>'numberReturned')::int + ), + 1, + 'Duplicate id pagination: traveling backward via prev token successfully loads 1 item. #392' +); + +-- ASC sorting: verify page 2 works when the timeline is inverted. +SELECT is( + ( + WITH p1 AS (SELECT search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1, "sortby": [{"field": "datetime", "direction": "asc"}]}'::jsonb)::jsonb AS j), + nt AS (SELECT split_part(jsonb_path_query_first((SELECT j FROM p1), '$.links[*] ? (@.rel == "next").href')->>0, 'token=', 2) AS t) + SELECT (search('{"ids": ["pgstac-test-item-duplicated"], "limit": 1, "sortby": [{"field": "datetime", "direction": "asc"}]}'::jsonb || jsonb_build_object('token', (SELECT t FROM nt)))->>'numberReturned')::int + ), + 1, + 'Duplicate id pagination: page 2 successfully steps forward when using ASC sort direction. #392' +); diff --git a/src/pypgstac/src/pypgstac/load.py b/src/pypgstac/src/pypgstac/load.py index 657580cb..f7680685 100644 --- a/src/pypgstac/src/pypgstac/load.py +++ b/src/pypgstac/src/pypgstac/load.py @@ -17,6 +17,7 @@ Iterable, Iterator, TextIO, + cast, ) import orjson @@ -109,7 +110,7 @@ def open_std( fh = stream.buffer if "b" in mode else stream close = False else: - fh = open(filename, mode, *args, **kwargs) + fh = cast("TextIO | BinaryIO", open(filename, mode, *args, **kwargs)) close = True try: