From 7c2eff731b862dc11b0eba7d246c97143cb5df94 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 29 Jun 2026 10:02:29 -0500 Subject: [PATCH 01/11] Add golf features to vector tiles --- images/tiler-imposm/config/layers/landuse_areas.json | 12 ++++++++++++ .../tiler-imposm/config/layers/landuse_points.json | 9 +++++++++ images/tiler-imposm/queries/ohm_mviews/buildings.sql | 3 ++- .../queries/ohm_mviews/transport_lines.sql | 2 ++ images/tiler-imposm/queries/ohm_mviews/water.sql | 2 +- 5 files changed, 26 insertions(+), 2 deletions(-) diff --git a/images/tiler-imposm/config/layers/landuse_areas.json b/images/tiler-imposm/config/layers/landuse_areas.json index ff427e8ea..c7d6e2ff0 100644 --- a/images/tiler-imposm/config/layers/landuse_areas.json +++ b/images/tiler-imposm/config/layers/landuse_areas.json @@ -87,6 +87,18 @@ "exclude": [ "water" ] + }, + "golf": { + "mapping": { + "golf": [ + "green", + "bunker", + "fairway", + "rough", + "tee", + "driving_range" + ] + } } } } diff --git a/images/tiler-imposm/config/layers/landuse_points.json b/images/tiler-imposm/config/layers/landuse_points.json index 2374ebff2..786a523fa 100644 --- a/images/tiler-imposm/config/layers/landuse_points.json +++ b/images/tiler-imposm/config/layers/landuse_points.json @@ -79,6 +79,15 @@ "__any__" ] } + }, + "golf": { + "mapping": { + "golf": [ + "pin", + "hole", + "tee" + ] + } } } } diff --git a/images/tiler-imposm/queries/ohm_mviews/buildings.sql b/images/tiler-imposm/queries/ohm_mviews/buildings.sql index 350b37219..b43b26e75 100644 --- a/images/tiler-imposm/queries/ohm_mviews/buildings.sql +++ b/images/tiler-imposm/queries/ohm_mviews/buildings.sql @@ -54,7 +54,8 @@ SELECT create_areas_mview( '(class = ''building:part'') AS is_building_part, EXISTS (SELECT 1 FROM osm_buildings_relation_members obrm WHERE obrm.member = ABS(osm_buildings.osm_id) AND obrm.role = ''outline'') AS hide_3d, render_height(height, building_height, building_levels) AS render_height, - render_min_height(min_height, building_min_level) AS render_min_height', + render_min_height(min_height, building_min_level) AS render_min_height, + NULLIF(tags->''golf'', '''') AS golf', '{"roof_height": "parse_to_meters(roof_height)"}'::jsonb, ARRAY['height', 'min_height', 'building_height', 'building_levels', 'building_min_level'] ); diff --git a/images/tiler-imposm/queries/ohm_mviews/transport_lines.sql b/images/tiler-imposm/queries/ohm_mviews/transport_lines.sql index e8ab79423..51b924aa4 100644 --- a/images/tiler-imposm/queries/ohm_mviews/transport_lines.sql +++ b/images/tiler-imposm/queries/ohm_mviews/transport_lines.sql @@ -215,6 +215,7 @@ BEGIN ELSE type END AS type, NULLIF(tags->'construction', '') AS construction, + NULLIF(tags->'golf', '') AS golf, class, NULLIF(name, '') AS name, NULLIF(tunnel, 0) AS tunnel, @@ -262,6 +263,7 @@ BEGIN ELSE type END AS type, NULLIF(tags->'construction', '') AS construction, + NULLIF(tags->'golf', '') AS golf, class, NULLIF(name, '') AS name, NULLIF(tunnel, 0) AS tunnel, diff --git a/images/tiler-imposm/queries/ohm_mviews/water.sql b/images/tiler-imposm/queries/ohm_mviews/water.sql index ac7e764a1..6dc4e3750 100644 --- a/images/tiler-imposm/queries/ohm_mviews/water.sql +++ b/images/tiler-imposm/queries/ohm_mviews/water.sql @@ -6,7 +6,7 @@ -- Delete existing views, in cascade DROP MATERIALIZED VIEW IF EXISTS mv_water_areas_z16_20 CASCADE; -SELECT create_areas_mview('osm_water_areas', 'mv_water_areas_z16_20', 0, 0, 'id, osm_id, type', NULL, NULL, NULL); +SELECT create_areas_mview('osm_water_areas', 'mv_water_areas_z16_20', 0, 0, 'id, osm_id, type', NULL, 'NULLIF(tags->''golf'', '''') AS golf', NULL); SELECT create_area_mview_from_mview('mv_water_areas_z16_20','mv_water_areas_z13_15',5,0.0,NULL); SELECT create_area_mview_from_mview('mv_water_areas_z13_15','mv_water_areas_z10_12',20,100, 'type IN (''water'',''pond'',''basin'',''canal'',''mill_pond'',''riverbank'')'); SELECT create_area_mview_from_mview('mv_water_areas_z10_12','mv_water_areas_z8_9',100,10000, NULL); From c35a19e672aa490a7a67d5383d98c00c5a301a24 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 29 Jun 2026 10:36:36 -0500 Subject: [PATCH 02/11] Add place=island to tiles --- images/tiler-imposm/queries/ohm_mviews/places.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/tiler-imposm/queries/ohm_mviews/places.sql b/images/tiler-imposm/queries/ohm_mviews/places.sql index 5091f060e..f0949dcba 100644 --- a/images/tiler-imposm/queries/ohm_mviews/places.sql +++ b/images/tiler-imposm/queries/ohm_mviews/places.sql @@ -195,7 +195,7 @@ SELECT create_place_points_centroids_mview ( SELECT create_place_points_centroids_mview ( 'mv_place_points_centroids_z11_20', - ARRAY ['plot', 'square', 'islet'], + ARRAY ['plot', 'square', 'islet', 'island'], ARRAY ['state', 'province', 'region', 'county', 'municipality', 'city', 'town', 'village', 'suburb', 'locality', 'hamlet', 'islet', 'neighbourhood', 'district', 'borough', 'quarter', 'isolated_dwelling', 'farm'] ); @@ -204,7 +204,7 @@ SELECT create_place_points_centroids_mview ( -- ============================================================================ SELECT create_place_areas_mview( 'mv_place_areas_z14_20', - ARRAY['plot', 'square', 'islet'] + ARRAY['plot', 'square', 'islet', 'island'] ); -- Refresh centroids views From 8684e92a28f68ead569bef23030cd54b1a70c112 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 29 Jun 2026 11:51:56 -0500 Subject: [PATCH 03/11] Fix routes showing outside their date range --- .../ohm_mviews/routes_01_merge_by_date.sql | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/images/tiler-imposm/queries/ohm_mviews/routes_01_merge_by_date.sql b/images/tiler-imposm/queries/ohm_mviews/routes_01_merge_by_date.sql index 9073b182b..df1800659 100644 --- a/images/tiler-imposm/queries/ohm_mviews/routes_01_merge_by_date.sql +++ b/images/tiler-imposm/queries/ohm_mviews/routes_01_merge_by_date.sql @@ -95,16 +95,14 @@ SELECT log_notice('STEP 3: Backfill for osm_route_multilines'); SET statement_timeout = 2400000; UPDATE osm_route_multilines SET start_decdate = isodatetodecimaldate(pad_date(start_date::TEXT, 'start')::TEXT, FALSE), - end_decdate = isodatetodecimaldate(pad_date(end_date::TEXT, 'end')::TEXT, FALSE) -WHERE ST_GeometryType(geometry) IN ('ST_LineString', 'ST_MultiLineString'); + end_decdate = isodatetodecimaldate(pad_date(end_date::TEXT, 'end')::TEXT, FALSE); --- osm_route_lines SET statement_timeout = 2400000; UPDATE osm_route_lines SET start_decdate = isodatetodecimaldate(pad_date(start_date::TEXT, 'start')::TEXT, FALSE), - end_decdate = isodatetodecimaldate(pad_date(end_date::TEXT, 'end')::TEXT, FALSE) -WHERE ST_GeometryType(geometry) IN ('ST_LineString', 'ST_MultiLineString'); + end_decdate = isodatetodecimaldate(pad_date(end_date::TEXT, 'end')::TEXT, FALSE); -- ============================================================================ -- STEP 4: Create Materialized View for merged routes per continuous temporal range @@ -124,6 +122,12 @@ WITH union_sources AS ( me_highway AS highway FROM osm_route_multilines WHERE geometry IS NOT NULL + -- Fail-safe: drop rows whose date is present but invalid (text exists but did not + -- parse to a decimal date). An absent date stays NULL and keeps its unbounded + -- meaning; an invalid date would otherwise become unbounded and render the route + -- across all time (see issues #1369 / #1362). Hide it until the data is fixed. + AND NOT (COALESCE(start_date, '') <> '' AND start_decdate IS NULL) + AND NOT (COALESCE(end_date, '') <> '' AND end_decdate IS NULL) UNION ALL @@ -136,6 +140,9 @@ WITH union_sources AS ( highway FROM osm_route_lines WHERE geometry IS NOT NULL + -- Fail-safe: drop rows whose date is present but invalid (see multi-lines branch above). + AND NOT (COALESCE(start_date, '') <> '' AND start_decdate IS NULL) + AND NOT (COALESCE(end_date, '') <> '' AND end_decdate IS NULL) ), -- CTE to identify ways that DO have at least one date ways_with_dates AS ( From bdba337b06f2c2e3728492100628c1ebf2271eb9 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 29 Jun 2026 13:06:28 -0500 Subject: [PATCH 04/11] Deploy vtiles per branch --- .github/workflows/preview-tiler-down.yaml | 86 ++++++++++++ .github/workflows/preview-tiler-up.yaml | 159 ++++++++++++++++++++++ values.k3s.preview-tiler.template.yaml | 129 ++++++++++++++++++ 3 files changed, 374 insertions(+) create mode 100644 .github/workflows/preview-tiler-down.yaml create mode 100644 .github/workflows/preview-tiler-up.yaml create mode 100644 values.k3s.preview-tiler.template.yaml diff --git a/.github/workflows/preview-tiler-down.yaml b/.github/workflows/preview-tiler-down.yaml new file mode 100644 index 000000000..f224b89c7 --- /dev/null +++ b/.github/workflows/preview-tiler-down.yaml @@ -0,0 +1,86 @@ +name: Preview tiler down (k3s) + +# Remove a tiler preview: helm uninstall + delete its ingress and released PVs. +# The host disks (/mnt/tiler-db-data-, /mnt/tiler-imposm-data-) use +# Retain PVs, so the data stays on the node after this runs. To reclaim disk or +# rebuild from scratch, delete those dirs on the node manually. +on: + workflow_dispatch: + inputs: + ref: + description: vtiles-* branch whose preview to remove + required: true + delete: + pull_request: + types: [closed] + +jobs: + teardown: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - name: Resolve branch + id: r + run: | + case "${{ github.event_name }}" in + workflow_dispatch) BRANCH="${{ github.event.inputs.ref }}" ;; + delete) BRANCH="${{ github.event.ref }}" ;; + pull_request) BRANCH="${{ github.event.pull_request.head.ref }}" ;; + esac + case "$BRANCH" in + vtiles-*) ;; + *) echo "branch '$BRANCH' is not a vtiles-* preview; nothing to do"; echo "skip=true" >> $GITHUB_OUTPUT; exit 0 ;; + esac + SLUG="$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' \ + | sed 's/[^a-z0-9]/-/g; s/-\+/-/g; s/^-//; s/-$//' | cut -c1-40 | sed 's/-$//')" + echo "slug=$SLUG" >> $GITHUB_OUTPUT + echo "release=tiler-$SLUG" >> $GITHUB_OUTPUT + echo "namespace=preview" >> $GITHUB_OUTPUT + + - name: Install cloudflared + if: steps.r.outputs.skip != 'true' + run: | + sudo curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 \ + -o /usr/local/bin/cloudflared + sudo chmod +x /usr/local/bin/cloudflared + + - name: Install helm + if: steps.r.outputs.skip != 'true' + uses: azure/setup-helm@v4 + with: + version: v3.15.1 + + - name: Setup kubeconfig + if: steps.r.outputs.skip != 'true' + run: | + mkdir -p $HOME/.kube + echo "${{ secrets.STAGING_K3S_KUBECONFIG }}" | base64 -d > $HOME/.kube/config + chmod 600 $HOME/.kube/config + + - name: Open Cloudflare Access tunnel to k3s API + if: steps.r.outputs.skip != 'true' + env: + TUNNEL_SERVICE_TOKEN_ID: ${{ secrets.STAGING_CF_ACCESS_CLIENT_ID }} + TUNNEL_SERVICE_TOKEN_SECRET: ${{ secrets.STAGING_CF_ACCESS_CLIENT_SECRET }} + run: | + cloudflared access tcp --hostname k3s.ohmstaging.org --url 127.0.0.1:16443 & + for i in {1..30}; do + if curl -sk -o /dev/null --max-time 5 https://127.0.0.1:16443/livez; then + echo "tunnel up"; exit 0 + fi + sleep 2 + done + echo "Tunnel failed to reach k3s" >&2; exit 1 + + - name: Teardown + if: steps.r.outputs.skip != 'true' + run: | + NS=${{ steps.r.outputs.namespace }} + REL=${{ steps.r.outputs.release }} + kubectl -n "$NS" delete ingress "$REL-ingress" --ignore-not-found + helm -n "$NS" uninstall "$REL" || echo "release $REL not found" + # delete the Retain PVs left behind (data on disk is NOT removed) + kubectl delete pv -l "app.kubernetes.io/instance=$REL" --ignore-not-found + echo "Done. Host disks /mnt/tiler-db-data-${{ steps.r.outputs.slug }} and" + echo "/mnt/tiler-imposm-data-${{ steps.r.outputs.slug }} remain on the node." + echo "rm them manually to reclaim space or rebuild from scratch." diff --git a/.github/workflows/preview-tiler-up.yaml b/.github/workflows/preview-tiler-up.yaml new file mode 100644 index 000000000..72ad91ea1 --- /dev/null +++ b/.github/workflows/preview-tiler-up.yaml @@ -0,0 +1,159 @@ +name: Preview tiler (k3s) + +on: + workflow_dispatch: + inputs: + ref: + description: vtiles-* branch to preview + required: true + push: + branches: + - 'vtiles-*' + +concurrency: + # one run per branch; a new push cancels the in-flight preview build + group: preview-tiler-${{ github.event.inputs.ref || github.ref_name }} + cancel-in-progress: true + +jobs: + preview: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Resolve names + id: n + run: | + BRANCH="${{ github.event.inputs.ref || github.ref_name }}" + case "$BRANCH" in + vtiles-*) ;; + *) echo "::error::branch '$BRANCH' must start with vtiles-"; exit 1 ;; + esac + # DNS-safe slug: lowercase, non-alnum -> '-', trim, max 40 chars. + SLUG="$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' \ + | sed 's/[^a-z0-9]/-/g; s/-\+/-/g; s/^-//; s/-$//' | cut -c1-40 | sed 's/-$//')" + echo "branch=$BRANCH" >> $GITHUB_OUTPUT + echo "slug=$SLUG" >> $GITHUB_OUTPUT + echo "release=tiler-$SLUG" >> $GITHUB_OUTPUT + echo "namespace=preview" >> $GITHUB_OUTPUT + echo "host=tiler-$SLUG.ohmstaging.org" >> $GITHUB_OUTPUT + echo "martin_svc=tiler-$SLUG-tiler-server-martin" >> $GITHUB_OUTPUT + + - uses: actions/checkout@v4 + with: + ref: ${{ steps.n.outputs.branch }} + fetch-depth: 0 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GHCR_GITHUB_TOKEN }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Setup git + run: | + git config --global user.email "noreply@developmentseed.org" + git config --global user.name "Github Action" + + - name: Install chartpress + run: pip install chartpress==2.3.0 ruamel.yaml + + - name: Run chartpress (build + push) + env: + GITHUB_TOKEN: ${{ secrets.GHCR_GITHUB_TOKEN }} + run: chartpress --push + + - name: Install cloudflared + run: | + sudo curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 \ + -o /usr/local/bin/cloudflared + sudo chmod +x /usr/local/bin/cloudflared + + - name: Install helm + uses: azure/setup-helm@v4 + with: + version: v3.15.1 + + - name: Setup kubeconfig + run: | + mkdir -p $HOME/.kube + KCFG="${{ secrets.STAGING_K3S_KUBECONFIG }}" + if [ -z "$KCFG" ]; then + echo "ERROR: STAGING_K3S_KUBECONFIG is empty" >&2; exit 1 + fi + echo "$KCFG" | base64 -d > $HOME/.kube/config + chmod 600 $HOME/.kube/config + + - name: Open Cloudflare Access tunnel to k3s API + env: + TUNNEL_SERVICE_TOKEN_ID: ${{ secrets.STAGING_CF_ACCESS_CLIENT_ID }} + TUNNEL_SERVICE_TOKEN_SECRET: ${{ secrets.STAGING_CF_ACCESS_CLIENT_SECRET }} + run: | + cloudflared access tcp --hostname k3s.ohmstaging.org --url 127.0.0.1:16443 & + for i in {1..30}; do + if curl -sk -o /dev/null --max-time 5 https://127.0.0.1:16443/livez; then + echo "tunnel up (k3s reachable)"; exit 0 + fi + sleep 2 + done + echo "Tunnel failed to reach k3s" >&2; exit 1 + + - name: Verify access + run: kubectl get nodes + + - name: Substitute secrets into preview-tiler values + uses: bluwy/substitute-string-action@v3 + with: + _input-file: 'values.k3s.preview-tiler.template.yaml' + _format-key: '{{key}}' + _output-file: 'values.k3s.preview-tiler.yaml' + SLUG: ${{ steps.n.outputs.slug }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Helm dep up + run: cd ohm && helm dep up + + - name: Deploy tiler preview + run: | + helm upgrade --install ${{ steps.n.outputs.release }} ./ohm \ + -n ${{ steps.n.outputs.namespace }} --create-namespace \ + -f ./ohm/values.yaml \ + -f ./values.k3s.preview-tiler.yaml + # no --wait: the imposm full import runs in the background + + - name: Expose martin via Ingress + run: | + cat <` in the `preview` namespace by +# preview-tiler-up.yaml. It builds an independent tiler stack (db + imposm + +# martin) on its own host disks, so it never touches the live tiler: +# /mnt/tiler-db-data- (postgres data) +# /mnt/tiler-imposm-data- (imposm cache + replication state) +# +# imposm runs a full import from the PBF with the branch's mappings, then +# builds the mviews. Martin is exposed at tiler-.ohmstaging.org through a +# standalone Ingress created by the workflow (the chart does not template an +# ingress for martin). +# +# Placeholders: {{SLUG}} {{AWS_ACCESS_KEY_ID}} {{AWS_SECRET_ACCESS_KEY}} +osm-seed: + environment: staging + cloudProvider: k3s + serviceType: ClusterIP + createClusterIssuer: false + ingressClassName: "" + domain: ohmstaging.org + adminEmail: admin@openhistoricalmap.org + + # --- everything off except the tiler build chain --- + web: { enabled: false } + db: { enabled: false } + memcached: { enabled: false } + cgimap: { enabled: false } + fullHistory: { enabled: false } + changesetsDump: { enabled: false } + dbBackupRestore: { enabled: false } + planetDump: { enabled: false } + replicationJob: { enabled: false } + populateApidb: { enabled: false } + osmProcessor: { enabled: false } + tilerServer: { enabled: false } + tilerVarnish: { enabled: false } + tilerServerCacheCleaner: { enabled: false } + tilerCacheCleanerJob: { enabled: false } + tilerCache: { enabled: false } + tilerMonitorPipeline: { enabled: false } + tmDb: { enabled: false } + tmApi: { enabled: false } + osmSimpleMetrics: { enabled: false } + monitoringReplication: { enabled: false } + changesetReplicationJob: { enabled: false } + planetFiles: { enabled: false } + taginfoWeb: { enabled: false } + taginfoDataProcessor: { enabled: false } + nominatimUI: { enabled: false } + nominatimApi: { enabled: false } + overpassApi: { enabled: false } + osmchaWeb: { enabled: false } + osmchaApi: { enabled: false } + osmchaDb: { enabled: false } + osmxAdiffBuilder: { enabled: false } + planetStats: { enabled: false } + + # --- tiler build chain ON, on per-branch disks --- + tilerDb: + enabled: true + priorityClass: medium-priority + env: + POSTGRES_DB: tiler_osm_production + POSTGRES_USER: postgres + POSTGRES_PASSWORD: "123456789" + POSTGRES_PORT: 5432 + sharedMemorySize: 2Gi + persistenceDisk: + enabled: true + accessMode: ReadWriteOnce + mountPath: /var/lib/postgresql/data + subPath: postgresql-data + staticHostPath: true + localVolumeHostPath: /mnt/tiler-db-data-{{SLUG}} + resources: + enabled: false + + tilerImposm: + enabled: true + priorityClass: medium-priority + serviceAccount: + enabled: false + env: + TILER_IMPORT_FROM: osm + TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260531_0302.osm.pbf + REPLICATION_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute/ + SEQUENCE_NUMBER: "1690000" + OVERWRITE_STATE: false + # preview must not write to the shared S3 expiry/language buckets + UPLOAD_EXPIRED_FILES: false + IMPORT_NATURAL_EARTH: true + IMPORT_OSM_LAND: true + IMPOSM3_IMPORT_LAYERS: "all" + CLOUDPROVIDER: aws + AWS_S3_BUCKET: "" + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" + REFRESH_MVIEWS: true + RECREATE_MVIEWS_ON_UPDATE: false + persistenceDisk: + enabled: true + accessMode: ReadWriteOnce + mountPath: /mnt/data + staticHostPath: true + localVolumeHostPath: /mnt/tiler-imposm-data-{{SLUG}} + resources: + enabled: true + nodeSelector: + enabled: false + nodeAffinity: + enabled: false + + tilerServerMartin: + enabled: true + priorityClass: medium-priority + replicaCount: 1 + env: + OHM_DOMAIN: ohmstaging.org + MARTIN_WORKER_PROCESSES: 8 + MARTIN_POOL_SIZE: 10 + # empty bucket = skip the vtiles_languages.geojson upload to S3 + AWS_S3_BUCKET: "" + resources: + enabled: false + nodeSelector: + enabled: false + nodeAffinity: + enabled: false From ed2399e3f92fc58dcafeac20f4285ed25e601bc7 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 29 Jun 2026 14:58:27 -0500 Subject: [PATCH 05/11] Update pipeline to deploy preview --- .github/workflows/preview-tiler-down.yaml | 86 ---------- .github/workflows/preview-tiler-up.yaml | 159 ------------------ .gitignore | 3 +- ...late.yaml => values.k3s.preview-tiler.yaml | 31 ++-- 4 files changed, 14 insertions(+), 265 deletions(-) delete mode 100644 .github/workflows/preview-tiler-down.yaml delete mode 100644 .github/workflows/preview-tiler-up.yaml rename values.k3s.preview-tiler.template.yaml => values.k3s.preview-tiler.yaml (77%) diff --git a/.github/workflows/preview-tiler-down.yaml b/.github/workflows/preview-tiler-down.yaml deleted file mode 100644 index f224b89c7..000000000 --- a/.github/workflows/preview-tiler-down.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: Preview tiler down (k3s) - -# Remove a tiler preview: helm uninstall + delete its ingress and released PVs. -# The host disks (/mnt/tiler-db-data-, /mnt/tiler-imposm-data-) use -# Retain PVs, so the data stays on the node after this runs. To reclaim disk or -# rebuild from scratch, delete those dirs on the node manually. -on: - workflow_dispatch: - inputs: - ref: - description: vtiles-* branch whose preview to remove - required: true - delete: - pull_request: - types: [closed] - -jobs: - teardown: - runs-on: ubuntu-22.04 - timeout-minutes: 20 - steps: - - name: Resolve branch - id: r - run: | - case "${{ github.event_name }}" in - workflow_dispatch) BRANCH="${{ github.event.inputs.ref }}" ;; - delete) BRANCH="${{ github.event.ref }}" ;; - pull_request) BRANCH="${{ github.event.pull_request.head.ref }}" ;; - esac - case "$BRANCH" in - vtiles-*) ;; - *) echo "branch '$BRANCH' is not a vtiles-* preview; nothing to do"; echo "skip=true" >> $GITHUB_OUTPUT; exit 0 ;; - esac - SLUG="$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' \ - | sed 's/[^a-z0-9]/-/g; s/-\+/-/g; s/^-//; s/-$//' | cut -c1-40 | sed 's/-$//')" - echo "slug=$SLUG" >> $GITHUB_OUTPUT - echo "release=tiler-$SLUG" >> $GITHUB_OUTPUT - echo "namespace=preview" >> $GITHUB_OUTPUT - - - name: Install cloudflared - if: steps.r.outputs.skip != 'true' - run: | - sudo curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 \ - -o /usr/local/bin/cloudflared - sudo chmod +x /usr/local/bin/cloudflared - - - name: Install helm - if: steps.r.outputs.skip != 'true' - uses: azure/setup-helm@v4 - with: - version: v3.15.1 - - - name: Setup kubeconfig - if: steps.r.outputs.skip != 'true' - run: | - mkdir -p $HOME/.kube - echo "${{ secrets.STAGING_K3S_KUBECONFIG }}" | base64 -d > $HOME/.kube/config - chmod 600 $HOME/.kube/config - - - name: Open Cloudflare Access tunnel to k3s API - if: steps.r.outputs.skip != 'true' - env: - TUNNEL_SERVICE_TOKEN_ID: ${{ secrets.STAGING_CF_ACCESS_CLIENT_ID }} - TUNNEL_SERVICE_TOKEN_SECRET: ${{ secrets.STAGING_CF_ACCESS_CLIENT_SECRET }} - run: | - cloudflared access tcp --hostname k3s.ohmstaging.org --url 127.0.0.1:16443 & - for i in {1..30}; do - if curl -sk -o /dev/null --max-time 5 https://127.0.0.1:16443/livez; then - echo "tunnel up"; exit 0 - fi - sleep 2 - done - echo "Tunnel failed to reach k3s" >&2; exit 1 - - - name: Teardown - if: steps.r.outputs.skip != 'true' - run: | - NS=${{ steps.r.outputs.namespace }} - REL=${{ steps.r.outputs.release }} - kubectl -n "$NS" delete ingress "$REL-ingress" --ignore-not-found - helm -n "$NS" uninstall "$REL" || echo "release $REL not found" - # delete the Retain PVs left behind (data on disk is NOT removed) - kubectl delete pv -l "app.kubernetes.io/instance=$REL" --ignore-not-found - echo "Done. Host disks /mnt/tiler-db-data-${{ steps.r.outputs.slug }} and" - echo "/mnt/tiler-imposm-data-${{ steps.r.outputs.slug }} remain on the node." - echo "rm them manually to reclaim space or rebuild from scratch." diff --git a/.github/workflows/preview-tiler-up.yaml b/.github/workflows/preview-tiler-up.yaml deleted file mode 100644 index 72ad91ea1..000000000 --- a/.github/workflows/preview-tiler-up.yaml +++ /dev/null @@ -1,159 +0,0 @@ -name: Preview tiler (k3s) - -on: - workflow_dispatch: - inputs: - ref: - description: vtiles-* branch to preview - required: true - push: - branches: - - 'vtiles-*' - -concurrency: - # one run per branch; a new push cancels the in-flight preview build - group: preview-tiler-${{ github.event.inputs.ref || github.ref_name }} - cancel-in-progress: true - -jobs: - preview: - runs-on: ubuntu-22.04 - timeout-minutes: 30 - steps: - - name: Resolve names - id: n - run: | - BRANCH="${{ github.event.inputs.ref || github.ref_name }}" - case "$BRANCH" in - vtiles-*) ;; - *) echo "::error::branch '$BRANCH' must start with vtiles-"; exit 1 ;; - esac - # DNS-safe slug: lowercase, non-alnum -> '-', trim, max 40 chars. - SLUG="$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' \ - | sed 's/[^a-z0-9]/-/g; s/-\+/-/g; s/^-//; s/-$//' | cut -c1-40 | sed 's/-$//')" - echo "branch=$BRANCH" >> $GITHUB_OUTPUT - echo "slug=$SLUG" >> $GITHUB_OUTPUT - echo "release=tiler-$SLUG" >> $GITHUB_OUTPUT - echo "namespace=preview" >> $GITHUB_OUTPUT - echo "host=tiler-$SLUG.ohmstaging.org" >> $GITHUB_OUTPUT - echo "martin_svc=tiler-$SLUG-tiler-server-martin" >> $GITHUB_OUTPUT - - - uses: actions/checkout@v4 - with: - ref: ${{ steps.n.outputs.branch }} - fetch-depth: 0 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GHCR_GITHUB_TOKEN }} - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Setup git - run: | - git config --global user.email "noreply@developmentseed.org" - git config --global user.name "Github Action" - - - name: Install chartpress - run: pip install chartpress==2.3.0 ruamel.yaml - - - name: Run chartpress (build + push) - env: - GITHUB_TOKEN: ${{ secrets.GHCR_GITHUB_TOKEN }} - run: chartpress --push - - - name: Install cloudflared - run: | - sudo curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 \ - -o /usr/local/bin/cloudflared - sudo chmod +x /usr/local/bin/cloudflared - - - name: Install helm - uses: azure/setup-helm@v4 - with: - version: v3.15.1 - - - name: Setup kubeconfig - run: | - mkdir -p $HOME/.kube - KCFG="${{ secrets.STAGING_K3S_KUBECONFIG }}" - if [ -z "$KCFG" ]; then - echo "ERROR: STAGING_K3S_KUBECONFIG is empty" >&2; exit 1 - fi - echo "$KCFG" | base64 -d > $HOME/.kube/config - chmod 600 $HOME/.kube/config - - - name: Open Cloudflare Access tunnel to k3s API - env: - TUNNEL_SERVICE_TOKEN_ID: ${{ secrets.STAGING_CF_ACCESS_CLIENT_ID }} - TUNNEL_SERVICE_TOKEN_SECRET: ${{ secrets.STAGING_CF_ACCESS_CLIENT_SECRET }} - run: | - cloudflared access tcp --hostname k3s.ohmstaging.org --url 127.0.0.1:16443 & - for i in {1..30}; do - if curl -sk -o /dev/null --max-time 5 https://127.0.0.1:16443/livez; then - echo "tunnel up (k3s reachable)"; exit 0 - fi - sleep 2 - done - echo "Tunnel failed to reach k3s" >&2; exit 1 - - - name: Verify access - run: kubectl get nodes - - - name: Substitute secrets into preview-tiler values - uses: bluwy/substitute-string-action@v3 - with: - _input-file: 'values.k3s.preview-tiler.template.yaml' - _format-key: '{{key}}' - _output-file: 'values.k3s.preview-tiler.yaml' - SLUG: ${{ steps.n.outputs.slug }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Helm dep up - run: cd ohm && helm dep up - - - name: Deploy tiler preview - run: | - helm upgrade --install ${{ steps.n.outputs.release }} ./ohm \ - -n ${{ steps.n.outputs.namespace }} --create-namespace \ - -f ./ohm/values.yaml \ - -f ./values.k3s.preview-tiler.yaml - # no --wait: the imposm full import runs in the background - - - name: Expose martin via Ingress - run: | - cat <` in the `preview` namespace by -# preview-tiler-up.yaml. It builds an independent tiler stack (db + imposm + -# martin) on its own host disks, so it never touches the live tiler: -# /mnt/tiler-db-data- (postgres data) -# /mnt/tiler-imposm-data- (imposm cache + replication state) -# -# imposm runs a full import from the PBF with the branch's mappings, then -# builds the mviews. Martin is exposed at tiler-.ohmstaging.org through a -# standalone Ingress created by the workflow (the chart does not template an -# ingress for martin). -# -# Placeholders: {{SLUG}} {{AWS_ACCESS_KEY_ID}} {{AWS_SECRET_ACCESS_KEY}} osm-seed: environment: staging cloudProvider: k3s @@ -59,11 +45,15 @@ osm-seed: # --- tiler build chain ON, on per-branch disks --- tilerDb: enabled: true + image: + repository: ghcr.io/openhistoricalmap/tiler-db + tag: 0.0.1-0.dev.git.3389.h62b55811 priorityClass: medium-priority env: POSTGRES_DB: tiler_osm_production POSTGRES_USER: postgres - POSTGRES_PASSWORD: "123456789" + # injected at install time via --set from $POSTGRES_PASSWORD + POSTGRES_PASSWORD: "" POSTGRES_PORT: 5432 sharedMemorySize: 2Gi persistenceDisk: @@ -72,18 +62,21 @@ osm-seed: mountPath: /var/lib/postgresql/data subPath: postgresql-data staticHostPath: true - localVolumeHostPath: /mnt/tiler-db-data-{{SLUG}} + localVolumeHostPath: /mnt/tiler-db-data-vtiles-features resources: enabled: false tilerImposm: enabled: true + image: + repository: ghcr.io/openhistoricalmap/tiler-imposm + tag: 0.0.1-0.dev.git.3408.h8684e92a priorityClass: medium-priority serviceAccount: enabled: false env: TILER_IMPORT_FROM: osm - TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260531_0302.osm.pbf + TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260629_0001.osm.pbf REPLICATION_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute/ SEQUENCE_NUMBER: "1690000" OVERWRITE_STATE: false @@ -103,7 +96,7 @@ osm-seed: accessMode: ReadWriteOnce mountPath: /mnt/data staticHostPath: true - localVolumeHostPath: /mnt/tiler-imposm-data-{{SLUG}} + localVolumeHostPath: /mnt/tiler-imposm-data-vtiles-features resources: enabled: true nodeSelector: @@ -112,7 +105,7 @@ osm-seed: enabled: false tilerServerMartin: - enabled: true + enabled: false priorityClass: medium-priority replicaCount: 1 env: From 2acd06cdc4e9ed28d492a400c272e8b6e3adf2d7 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 30 Jun 2026 09:49:11 -0500 Subject: [PATCH 06/11] Update imposm config for preview --- images/tiler-imposm/start.sh | 6 ++++++ ohm/requirements.yaml | 2 +- values.k3s.preview-tiler.yaml | 40 +++++++++++++++++++++++++++++------ 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh index 67170fa30..33f177420 100755 --- a/images/tiler-imposm/start.sh +++ b/images/tiler-imposm/start.sh @@ -125,6 +125,12 @@ function uploadExpiredFiles() { # Function to upload last state file function uploadLastState() { + # Skip when no bucket is configured (e.g. preview builds with empty + # AWS_S3_BUCKET). Without this, `aws s3 cp` runs with an empty target and + # fails every loop with "Invalid argument type". + if [ -z "$AWS_S3_BUCKET" ]; then + return + fi # Path to the last.state.txt file local state_file="$DIFF_DIR/last.state.txt" local s3_path="${AWS_S3_BUCKET}/${BUCKET_IMPOSM_FOLDER}/last.state.txt" diff --git a/ohm/requirements.yaml b/ohm/requirements.yaml index 1d8c400f6..864116e73 100644 --- a/ohm/requirements.yaml +++ b/ohm/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: osm-seed - version: '0.1.0-0.dev.git.1006.h3d2d82a' + version: '0.1.0-0.dev.git.997.h49b5861' repository: https://osm-seed.github.io/osm-seed-chart/ \ No newline at end of file diff --git a/values.k3s.preview-tiler.yaml b/values.k3s.preview-tiler.yaml index 8ca6fca72..c669de56d 100644 --- a/values.k3s.preview-tiler.yaml +++ b/values.k3s.preview-tiler.yaml @@ -46,7 +46,7 @@ osm-seed: tilerDb: enabled: true image: - repository: ghcr.io/openhistoricalmap/tiler-db + name: ghcr.io/openhistoricalmap/tiler-db tag: 0.0.1-0.dev.git.3389.h62b55811 priorityClass: medium-priority env: @@ -56,27 +56,55 @@ osm-seed: POSTGRES_PASSWORD: "" POSTGRES_PORT: 5432 sharedMemorySize: 2Gi + # Without this the db runs on postgres defaults (128MB shared_buffers, 64MB + # maintenance_work_mem), which makes a full import + building ~50 mviews very + # slow. maintenance_work_mem is the big lever for mview/index creation. + postgresqlConfig: + enabled: true + values: | + listen_addresses = '*' + max_connections = 100 + shared_buffers = 8GB + work_mem = 256MB + maintenance_work_mem = 4GB + effective_cache_size = 16GB + max_wal_size = 8GB + min_wal_size = 1GB + random_page_cost = 1.0 + checkpoint_completion_target = 0.9 + dynamic_shared_memory_type = posix + effective_io_concurrency = 300 + datestyle = 'iso, mdy' + timezone = 'Etc/UTC' + default_text_search_config = 'pg_catalog.english' + # Parallelism: faster import + mview/index builds + max_parallel_workers_per_gather = 4 + max_parallel_maintenance_workers = 4 + max_parallel_workers = 8 + max_worker_processes = 8 persistenceDisk: enabled: true accessMode: ReadWriteOnce mountPath: /var/lib/postgresql/data subPath: postgresql-data staticHostPath: true - localVolumeHostPath: /mnt/tiler-db-data-vtiles-features + localVolumeHostPath: /mnt/tiler-db-data-v2 resources: enabled: false tilerImposm: enabled: true image: - repository: ghcr.io/openhistoricalmap/tiler-imposm + name: ghcr.io/openhistoricalmap/tiler-imposm tag: 0.0.1-0.dev.git.3408.h8684e92a priorityClass: medium-priority + livenessProbe: + initialDelaySeconds: 14400 serviceAccount: enabled: false env: TILER_IMPORT_FROM: osm - TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260629_0001.osm.pbf + TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260630_0001.osm.pbf REPLICATION_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute/ SEQUENCE_NUMBER: "1690000" OVERWRITE_STATE: false @@ -90,13 +118,13 @@ osm-seed: AWS_ACCESS_KEY_ID: "" AWS_SECRET_ACCESS_KEY: "" REFRESH_MVIEWS: true - RECREATE_MVIEWS_ON_UPDATE: false + RECREATE_MVIEWS_ON_UPDATE: true persistenceDisk: enabled: true accessMode: ReadWriteOnce mountPath: /mnt/data staticHostPath: true - localVolumeHostPath: /mnt/tiler-imposm-data-vtiles-features + localVolumeHostPath: /mnt/tiler-imposm-data-v2 resources: enabled: true nodeSelector: From 3c8ddd29b57768ea126c501f353ff0e14a4beacb Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 2 Jul 2026 11:39:17 -0500 Subject: [PATCH 07/11] Update volume for tiler staging --- values.k3s.staging.template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values.k3s.staging.template.yaml b/values.k3s.staging.template.yaml index 2274f18f6..56ce8cb62 100644 --- a/values.k3s.staging.template.yaml +++ b/values.k3s.staging.template.yaml @@ -48,7 +48,7 @@ osm-seed: mountPath: /var/lib/postgresql/data subPath: postgresql-data staticHostPath: true - localVolumeHostPath: /mnt/tiler-db-data + localVolumeHostPath: /mnt/tiler-db-data-v2 resources: enabled: false postgresqlConfig: @@ -141,7 +141,7 @@ osm-seed: accessMode: ReadWriteOnce mountPath: /mnt/data staticHostPath: true - localVolumeHostPath: /mnt/tiler-imposm-data + localVolumeHostPath: /mnt/tiler-imposm-data-v2 resources: enabled: true nodeSelector: From ab2776d4d53c97272b3cf57ba5a14ae6abfcf96c Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 2 Jul 2026 15:58:50 -0500 Subject: [PATCH 08/11] Add natural_lines layer for cliff features --- .../config/layers/natural_lines.json | 63 +++++++++++++++++++ .../queries/ohm_mviews/natural_lines.sql | 18 ++++++ images/tiler-imposm/scripts/create_mviews.sh | 3 + images/tiler-imposm/scripts/refresh_mviews.sh | 8 +++ .../imposm_config_loader.py | 1 + .../tiler-monitor-pipeline/tables_config.json | 4 +- .../tiler-server-martin/config/functions.json | 11 ++++ 7 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 images/tiler-imposm/config/layers/natural_lines.json create mode 100644 images/tiler-imposm/queries/ohm_mviews/natural_lines.sql diff --git a/images/tiler-imposm/config/layers/natural_lines.json b/images/tiler-imposm/config/layers/natural_lines.json new file mode 100644 index 000000000..90b60622e --- /dev/null +++ b/images/tiler-imposm/config/layers/natural_lines.json @@ -0,0 +1,63 @@ +{ + "tags": { + "load_all": true, + "exclude": [ + "created_by", + "source", + "source:datetime" + ] + }, + "generalized_tables": {}, + "tables": { + "natural_lines": { + "fields": [ + { + "type": "id", + "name": "osm_id", + "key": null + }, + { + "type": "geometry", + "name": "geometry", + "key": null + }, + { + "type": "string", + "name": "name", + "key": "name" + }, + { + "type": "mapping_value", + "name": "type", + "key": null + }, + { + "type": "mapping_key", + "name": "class", + "key": null + }, + { + "type": "string", + "name": "start_date", + "key": "start_date" + }, + { + "type": "string", + "name": "end_date", + "key": "end_date" + }, + { + "type": "hstore_tags", + "name": "tags", + "key": null + } + ], + "type": "linestring", + "mapping": { + "natural": [ + "cliff" + ] + } + } + } +} diff --git a/images/tiler-imposm/queries/ohm_mviews/natural_lines.sql b/images/tiler-imposm/queries/ohm_mviews/natural_lines.sql new file mode 100644 index 000000000..fd04e7194 --- /dev/null +++ b/images/tiler-imposm/queries/ohm_mviews/natural_lines.sql @@ -0,0 +1,18 @@ +-- ============================================================================ +-- Natural Lines Materialized Views for Multiple Zoom Levels +-- Natural line features (natural=cliff, ...) split out of water_lines so they +-- are not tied to the water layer (issue #1345). Source table: osm_natural_lines. +-- The base type filter is kept explicit so new natural line types can be added +-- to the import without automatically showing up at every zoom. +-- ============================================================================ + +DROP MATERIALIZED VIEW IF EXISTS mv_natural_lines_z16_20 CASCADE; + +SELECT create_lines_mview('osm_natural_lines', 'mv_natural_lines_z16_20', 0, 0, 'id, osm_id, type', 'type IN (''cliff'')'); +SELECT create_mview_line_from_mview('mv_natural_lines_z16_20', 'mv_natural_lines_z13_15', 5, 'type IN (''cliff'')'); +SELECT create_mview_line_from_mview('mv_natural_lines_z13_15', 'mv_natural_lines_z10_12', 20, 'type IN (''cliff'')'); + +-- Refresh lines views +-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_natural_lines_z16_20; +-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_natural_lines_z13_15; +-- REFRESH MATERIALIZED VIEW CONCURRENTLY mv_natural_lines_z10_12; diff --git a/images/tiler-imposm/scripts/create_mviews.sh b/images/tiler-imposm/scripts/create_mviews.sh index e10632670..eec48f41b 100755 --- a/images/tiler-imposm/scripts/create_mviews.sh +++ b/images/tiler-imposm/scripts/create_mviews.sh @@ -85,6 +85,9 @@ execute_sql_file queries/ohm_mviews/transport_lines.sql execute_sql_file queries/ohm_mviews/water.sql +## natural lines (cliff, ...) — split out of water_lines (#1345) +execute_sql_file queries/ohm_mviews/natural_lines.sql + ## routes execute_sql_file queries/ohm_mviews/routes_01_merge_by_date.sql execute_sql_file queries/ohm_mviews/routes_02_indexed.sql diff --git a/images/tiler-imposm/scripts/refresh_mviews.sh b/images/tiler-imposm/scripts/refresh_mviews.sh index 824584a36..8c1355ac0 100755 --- a/images/tiler-imposm/scripts/refresh_mviews.sh +++ b/images/tiler-imposm/scripts/refresh_mviews.sh @@ -228,6 +228,13 @@ water_views=( mv_water_lines_z8_9 ) +# natural lines (cliff, ...) — split out of water_lines (#1345) +natural_views=( + mv_natural_lines_z16_20 + mv_natural_lines_z13_15 + mv_natural_lines_z10_12 +) + buildings_views=( # areas mv_buildings_areas_z14_15 @@ -293,6 +300,7 @@ refresh_mviews_group "OTHERS" 180 light "${others_views[@]}" & refresh_mviews_group "COMMUNICATION" 180 light "${communication_views[@]}" & refresh_mviews_group "PLACES" 180 light "${places_views[@]}" & refresh_mviews_group "WATER" 180 light "${water_views[@]}" & +refresh_mviews_group "NATURAL" 180 light "${natural_views[@]}" & refresh_mviews_group "BUILDINGS" 180 light "${buildings_views[@]}" & refresh_mviews_group "ROUTES" 180 light "${routes_views[@]}" & refresh_mviews_group "NO_ADMIN_BOUNDARIES" 180 light "${no_admin_boundaries_views[@]}" & diff --git a/images/tiler-monitor-pipeline/imposm_config_loader.py b/images/tiler-monitor-pipeline/imposm_config_loader.py index 1d85b0c3d..2061bbc32 100644 --- a/images/tiler-monitor-pipeline/imposm_config_loader.py +++ b/images/tiler-monitor-pipeline/imposm_config_loader.py @@ -45,6 +45,7 @@ "osm_landuse_points": ["mv_landuse_points", "mv_landuse_points_centroids_z6_7", "mv_landuse_points_centroids_z16_20"], "osm_water_areas": ["mv_water_areas_z0_2", "mv_water_areas_z16_20", "mv_water_areas_centroids_z8_9", "mv_water_areas_centroids_z16_20"], "osm_water_lines": ["mv_water_lines_z8_9", "mv_water_lines_z16_20"], + "osm_natural_lines": ["mv_natural_lines_z10_12", "mv_natural_lines_z16_20"], "osm_other_areas": ["mv_other_areas_z8_9", "mv_other_areas_z16_20"], "osm_other_lines": ["mv_other_lines_z14_15", "mv_other_lines_z16_20"], "osm_other_points": ["mv_other_points", "mv_other_points_centroids_z8_9", "mv_other_points_centroids_z16_20"], diff --git a/images/tiler-monitor-pipeline/tables_config.json b/images/tiler-monitor-pipeline/tables_config.json index ef0c407f1..dcd7fdafa 100644 --- a/images/tiler-monitor-pipeline/tables_config.json +++ b/images/tiler-monitor-pipeline/tables_config.json @@ -53,8 +53,8 @@ "views": ["mv_other_areas_z8_9", "mv_other_areas_z16_20", "mv_other_lines_z14_15", "mv_other_lines_z16_20", "mv_other_points", "mv_other_points_centroids_z8_9", "mv_other_points_centroids_z16_20"] }, "natural": { - "tables": ["osm_landuse_areas", "osm_landuse_lines", "osm_landuse_points", "osm_water_areas", "osm_water_lines"], - "views": ["mv_landuse_areas_z6_7", "mv_landuse_areas_z16_20", "mv_landuse_lines_z14_15", "mv_landuse_lines_z16_20", "mv_landuse_points", "mv_landuse_points_centroids_z6_7", "mv_landuse_points_centroids_z16_20", "mv_water_areas_z0_2", "mv_water_areas_z16_20", "mv_water_areas_centroids_z8_9", "mv_water_areas_centroids_z16_20", "mv_water_lines_z8_9", "mv_water_lines_z16_20"] + "tables": ["osm_landuse_areas", "osm_landuse_lines", "osm_landuse_points", "osm_water_areas", "osm_water_lines", "osm_natural_lines"], + "views": ["mv_landuse_areas_z6_7", "mv_landuse_areas_z16_20", "mv_landuse_lines_z14_15", "mv_landuse_lines_z16_20", "mv_landuse_points", "mv_landuse_points_centroids_z6_7", "mv_landuse_points_centroids_z16_20", "mv_water_areas_z0_2", "mv_water_areas_z16_20", "mv_water_areas_centroids_z8_9", "mv_water_areas_centroids_z16_20", "mv_water_lines_z8_9", "mv_water_lines_z16_20", "mv_natural_lines_z10_12", "mv_natural_lines_z16_20"] }, "place": { "tables": ["osm_place_areas", "osm_place_points"], diff --git a/images/tiler-server-martin/config/functions.json b/images/tiler-server-martin/config/functions.json index 52ff5d48b..79ef1b785 100644 --- a/images/tiler-server-martin/config/functions.json +++ b/images/tiler-server-martin/config/functions.json @@ -104,6 +104,17 @@ [null, "mv_water_lines_z16_20"] ] }, + { + "function_name": "natural_lines", + "source_layer": "natural_lines", + "exclude_columns": ["geometry", "tags"], + "min_zoom": 10, + "zoom_mapping": [ + [12, "mv_natural_lines_z10_12"], + [15, "mv_natural_lines_z13_15"], + [null, "mv_natural_lines_z16_20"] + ] + }, { "function_name": "transport_areas", "source_layer": "transport_areas", From 1e033974e2dd97fa2b86c2a774085f5006147ca1 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 2 Jul 2026 15:59:21 -0500 Subject: [PATCH 09/11] Remove cliff from water_lines layer --- images/tiler-imposm/config/layers/water_lines.json | 3 --- images/tiler-imposm/queries/ohm_mviews/water.sql | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/images/tiler-imposm/config/layers/water_lines.json b/images/tiler-imposm/config/layers/water_lines.json index 2604360e0..217926d40 100644 --- a/images/tiler-imposm/config/layers/water_lines.json +++ b/images/tiler-imposm/config/layers/water_lines.json @@ -64,9 +64,6 @@ ], "barrier": [ "ditch" - ], - "natural": [ - "cliff" ] } } diff --git a/images/tiler-imposm/queries/ohm_mviews/water.sql b/images/tiler-imposm/queries/ohm_mviews/water.sql index 6dc4e3750..1b91d5a60 100644 --- a/images/tiler-imposm/queries/ohm_mviews/water.sql +++ b/images/tiler-imposm/queries/ohm_mviews/water.sql @@ -27,9 +27,9 @@ select create_mview_centroid_from_mview('mv_water_areas_z8_9','mv_water_areas_ce -- Water lines Materialized Views for Multiple Zoom Levels -- ============================================================================ -SELECT create_lines_mview('osm_water_lines', 'mv_water_lines_z16_20', 0, 0, 'id, osm_id, type', 'type IN (''river'', ''canal'', ''cliff'', ''dam'', ''stream'', ''ditch'', ''drain'')'); -SELECT create_mview_line_from_mview('mv_water_lines_z16_20', 'mv_water_lines_z13_15', 5, 'type IN (''river'', ''canal'', ''cliff'', ''dam'', ''stream'')'); -SELECT create_mview_line_from_mview('mv_water_lines_z13_15', 'mv_water_lines_z10_12', 20, 'type IN (''river'', ''canal'', ''cliff'', ''dam'')'); +SELECT create_lines_mview('osm_water_lines', 'mv_water_lines_z16_20', 0, 0, 'id, osm_id, type', 'type IN (''river'', ''canal'', ''dam'', ''stream'', ''ditch'', ''drain'')'); +SELECT create_mview_line_from_mview('mv_water_lines_z16_20', 'mv_water_lines_z13_15', 5, 'type IN (''river'', ''canal'', ''dam'', ''stream'')'); +SELECT create_mview_line_from_mview('mv_water_lines_z13_15', 'mv_water_lines_z10_12', 20, 'type IN (''river'', ''canal'', ''dam'')'); SELECT create_mview_line_from_mview('mv_water_lines_z10_12', 'mv_water_lines_z8_9', 100, 'type IN (''river'', ''canal'')'); -- Refresh areas views From b6a77ee2bc1adab61da0f71ed50b8ff007395853 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Sun, 5 Jul 2026 08:56:57 -0500 Subject: [PATCH 10/11] Add tags->'golf' as gold in buildings --- images/tiler-imposm/queries/ohm_mviews/buildings.sql | 3 ++- ohm/requirements.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/images/tiler-imposm/queries/ohm_mviews/buildings.sql b/images/tiler-imposm/queries/ohm_mviews/buildings.sql index b43b26e75..823754b56 100644 --- a/images/tiler-imposm/queries/ohm_mviews/buildings.sql +++ b/images/tiler-imposm/queries/ohm_mviews/buildings.sql @@ -28,7 +28,8 @@ SELECT create_points_mview( 'FALSE::boolean AS hide_3d', 'NULL::text AS roof_material', 'NULL::text AS roof_colour', - 'NULL::text AS roof_shape' + 'NULL::text AS roof_shape', + 'NULLIF(tags->''golf'', '''') AS golf' ], NULL ); diff --git a/ohm/requirements.yaml b/ohm/requirements.yaml index 864116e73..c5a7cbf13 100644 --- a/ohm/requirements.yaml +++ b/ohm/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: osm-seed - version: '0.1.0-0.dev.git.997.h49b5861' + version: '0.1.0-0.dev.git.998.h0ce62c6' repository: https://osm-seed.github.io/osm-seed-chart/ \ No newline at end of file From e0e5c83788d9c3dbcacad92c5d4f475681dc2f97 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 6 Jul 2026 09:21:21 -0500 Subject: [PATCH 11/11] Enable livenessProbe in staging and production tiler --- values.k3s.preview-tiler.yaml | 60 ++++++++++++++++++++++------- values.k3s.production.template.yaml | 6 ++- values.k3s.staging.template.yaml | 2 + 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/values.k3s.preview-tiler.yaml b/values.k3s.preview-tiler.yaml index c669de56d..f375d2fbc 100644 --- a/values.k3s.preview-tiler.yaml +++ b/values.k3s.preview-tiler.yaml @@ -1,4 +1,4 @@ -osm-seed: +#osm-seed: environment: staging cloudProvider: k3s serviceType: ClusterIP @@ -20,7 +20,6 @@ osm-seed: populateApidb: { enabled: false } osmProcessor: { enabled: false } tilerServer: { enabled: false } - tilerVarnish: { enabled: false } tilerServerCacheCleaner: { enabled: false } tilerCacheCleanerJob: { enabled: false } tilerCache: { enabled: false } @@ -55,7 +54,7 @@ osm-seed: # injected at install time via --set from $POSTGRES_PASSWORD POSTGRES_PASSWORD: "" POSTGRES_PORT: 5432 - sharedMemorySize: 2Gi + sharedMemorySize: 4Gi # Without this the db runs on postgres defaults (128MB shared_buffers, 64MB # maintenance_work_mem), which makes a full import + building ~50 mviews very # slow. maintenance_work_mem is the big lever for mview/index creation. @@ -64,12 +63,12 @@ osm-seed: values: | listen_addresses = '*' max_connections = 100 - shared_buffers = 8GB - work_mem = 256MB - maintenance_work_mem = 4GB - effective_cache_size = 16GB - max_wal_size = 8GB - min_wal_size = 1GB + shared_buffers = 16GB + work_mem = 512MB + maintenance_work_mem = 10GB + effective_cache_size = 32GB + max_wal_size = 24GB + min_wal_size = 2GB random_page_cost = 1.0 checkpoint_completion_target = 0.9 dynamic_shared_memory_type = posix @@ -77,7 +76,13 @@ osm-seed: datestyle = 'iso, mdy' timezone = 'Etc/UTC' default_text_search_config = 'pg_catalog.english' - # Parallelism: faster import + mview/index builds + # Disposable preview import DB: trade crash durability for import speed. + # synchronous_commit=off only risks losing the last few txns on a crash + # (no corruption). full_page_writes=off is the bigger win but can leave + # torn pages after a crash; safe here only because we rebuild from scratch. + synchronous_commit = off + full_page_writes = off + # Parallelism: sized to the 8-core node (freed after staging tiler is gone) max_parallel_workers_per_gather = 4 max_parallel_maintenance_workers = 4 max_parallel_workers = 8 @@ -96,15 +101,15 @@ osm-seed: enabled: true image: name: ghcr.io/openhistoricalmap/tiler-imposm - tag: 0.0.1-0.dev.git.3408.h8684e92a + tag: 0.0.1-0.dev.git.3415.hb6a77ee2 priorityClass: medium-priority livenessProbe: - initialDelaySeconds: 14400 + enabled: false serviceAccount: enabled: false env: TILER_IMPORT_FROM: osm - TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260630_0001.osm.pbf + TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260704_0001.osm.pbf REPLICATION_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute/ SEQUENCE_NUMBER: "1690000" OVERWRITE_STATE: false @@ -119,6 +124,7 @@ osm-seed: AWS_SECRET_ACCESS_KEY: "" REFRESH_MVIEWS: true RECREATE_MVIEWS_ON_UPDATE: true + STARTUP_GRACE_SECONDS: "86400" persistenceDisk: enabled: true accessMode: ReadWriteOnce @@ -134,10 +140,15 @@ osm-seed: tilerServerMartin: enabled: false + image: + name: ghcr.io/openhistoricalmap/tiler-server-martin + tag: 0.0.1-0.dev.git.3413.hab2776d4 priorityClass: medium-priority replicaCount: 1 + # POSTGRES_* come from the tiler-db configmap (envFrom), so martin connects + # to the DB built by tilerImposm above. env: - OHM_DOMAIN: ohmstaging.org + OHM_DOMAIN: preview-tiler.ohmstaging.org MARTIN_WORKER_PROCESSES: 8 MARTIN_POOL_SIZE: 10 # empty bucket = skip the vtiles_languages.geojson upload to S3 @@ -148,3 +159,24 @@ osm-seed: enabled: false nodeAffinity: enabled: false + + # Cache in front of martin (VCL backend = -tiler-server-martin). + # Serves tiles on port 6081 at /maps/ohm/{z}/{x}/{y}. Ingress stays off: + # reach it with a port-forward. Set ingress.enabled + hosts to expose it. + tilerVarnish: + enabled: false + ingress: + enabled: false + image: + name: varnish + tag: "7.5" + priorityClass: medium-priority + replicaCount: 1 + dynamicSize: "4G" + staticSize: "1G" + resources: + enabled: false + nodeSelector: + enabled: false + nodeAffinity: + enabled: false diff --git a/values.k3s.production.template.yaml b/values.k3s.production.template.yaml index f2580da70..231019c00 100644 --- a/values.k3s.production.template.yaml +++ b/values.k3s.production.template.yaml @@ -48,7 +48,7 @@ osm-seed: mountPath: /var/lib/postgresql/data subPath: postgresql-data staticHostPath: true - localVolumeHostPath: /mnt/tiler-db-data + localVolumeHostPath: /mnt/tiler-db-data-v2 resources: enabled: false postgresqlConfig: @@ -135,6 +135,8 @@ osm-seed: tilerImposm: enabled: true priorityClass: medium-priority + livenessProbe: + enabled: true serviceAccount: enabled: false env: @@ -158,7 +160,7 @@ osm-seed: accessMode: ReadWriteOnce mountPath: /mnt/data staticHostPath: true - localVolumeHostPath: /mnt/tiler-imposm-data + localVolumeHostPath: /mnt/tiler-imposm-data-v2 resources: enabled: true nodeSelector: diff --git a/values.k3s.staging.template.yaml b/values.k3s.staging.template.yaml index 56ce8cb62..3015e699e 100644 --- a/values.k3s.staging.template.yaml +++ b/values.k3s.staging.template.yaml @@ -118,6 +118,8 @@ osm-seed: tilerImposm: enabled: true priorityClass: medium-priority + livenessProbe: + enabled: true serviceAccount: enabled: false env: