From 2914cb16808452c9e5b9c77d001098bb645d2587 Mon Sep 17 00:00:00 2001 From: Sebastian Madrid Ontiveros Date: Wed, 10 Jun 2026 18:53:23 +0100 Subject: [PATCH] Add SMO-Scottish-LiDAR Ruby gem to catalog Pure Ruby gem for listing and downloading Scottish Public Sector LiDAR data (DSM, DTM, LAZ) from the Registry of Open Data on AWS. Supports all survey phases (1-5) and Outer Hebrides. No external dependencies. MIT licensed. Co-Authored-By: Claude Opus 4.6 --- .../SMO-Scottish-LiDAR/.gitignore | 4 + .../SMO-Scottish-LiDAR/CHANGELOG.md | 20 ++ .../SMO-Scottish-LiDAR/CODE_OF_CONDUCT.md | 132 ++++++++ .../SMO-Scottish-LiDAR/Gemfile | 3 + .../SMO-Scottish-LiDAR/LICENSE | 21 ++ .../SMO-Scottish-LiDAR/README.md | 282 ++++++++++++++++++ .../SMO-Scottish-LiDAR/Rakefile | 9 + .../SMO-Scottish-LiDAR/examples/demo.rb | 94 ++++++ .../examples/download_batch_tiles.rb | 50 ++++ .../examples/download_individual_tile.rb | 11 + .../examples/outer_hebrides_dsm.rb | 18 ++ .../examples/outer_hebrides_dtm.rb | 18 ++ .../examples/outer_hebrides_laz.rb | 18 ++ .../examples/phase_1_dsm.rb | 16 + .../examples/phase_1_dtm.rb | 16 + .../examples/phase_1_laz.rb | 16 + .../examples/phase_2_dsm.rb | 16 + .../examples/phase_2_dtm.rb | 16 + .../examples/phase_2_laz.rb | 16 + .../examples/phase_3_dsm.rb | 16 + .../examples/phase_3_dtm.rb | 16 + .../examples/phase_3_laz.rb | 16 + .../examples/phase_4_dsm.rb | 16 + .../examples/phase_4_dtm.rb | 16 + .../examples/phase_4_laz.rb | 16 + .../examples/phase_5_dsm.rb | 16 + .../examples/phase_5_dtm.rb | 16 + .../examples/phase_5_laz.rb | 16 + .../lib/smo_scottish_lidar.rb | 7 + .../lib/smo_scottish_lidar/client.rb | 150 ++++++++++ .../lib/smo_scottish_lidar/constants.rb | 61 ++++ .../lib/smo_scottish_lidar/downloader.rb | 134 +++++++++ .../lib/smo_scottish_lidar/lister.rb | 75 +++++ .../lib/smo_scottish_lidar/version.rb | 5 + .../smo_scottish_lidar.gemspec | 30 ++ .../spec/smo_scottish_lidar_spec.rb | 100 +++++++ 36 files changed, 1482 insertions(+) create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/.gitignore create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/CHANGELOG.md create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/CODE_OF_CONDUCT.md create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/Gemfile create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/LICENSE create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/README.md create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/Rakefile create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/demo.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/download_batch_tiles.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/download_individual_tile.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_dsm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_dtm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_laz.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_dsm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_dtm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_laz.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_dsm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_dtm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_laz.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_dsm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_dtm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_laz.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_dsm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_dtm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_laz.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_dsm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_dtm.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_laz.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/client.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/constants.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/downloader.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/lister.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/version.rb create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/smo_scottish_lidar.gemspec create mode 100644 07 Ruby Exchange Gems/SMO-Scottish-LiDAR/spec/smo_scottish_lidar_spec.rb diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/.gitignore b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/.gitignore new file mode 100644 index 00000000..6c90ab81 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/.gitignore @@ -0,0 +1,4 @@ +*.gem +.ruby-lsp/ +.DS_Store +Gemfile.lock diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/CHANGELOG.md b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/CHANGELOG.md new file mode 100644 index 00000000..43eff9c1 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.0] - 2026-05-04 + +### Added +- `SmoScottishLidar::Client` for listing and downloading LiDAR data from AWS S3 +- Support for all survey phases (1–5) and Outer Hebrides +- DSM, DTM, and LAZ dataset types +- OS National Grid square filtering +- Paginated S3 listing with continuation tokens +- Streamed downloads with redirect following and progress callback +- Dry-run mode via verbose option +- RSpec test suite diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/CODE_OF_CONDUCT.md b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..67fe8cee --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/CODE_OF_CONDUCT.md @@ -0,0 +1,132 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[INSERT CONTACT METHOD]. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/Gemfile b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/Gemfile new file mode 100644 index 00000000..b4e2a20b --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/Gemfile @@ -0,0 +1,3 @@ +source "https://rubygems.org" + +gemspec diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/LICENSE b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/LICENSE new file mode 100644 index 00000000..a8c42a61 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Sebastian Madrid Ontiveros + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/README.md b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/README.md new file mode 100644 index 00000000..3d525b18 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/README.md @@ -0,0 +1,282 @@ +# smo_scottish_lidar + +[![Gem Version](https://img.shields.io/badge/gem-v0.1.0-1D9E75)](https://rubygems.org/gems/smo_scottish_lidar) +[![Ruby](https://img.shields.io/badge/ruby-%3E%3D%202.7-CC342D)](https://www.ruby-lang.org) +[![License: MIT](https://img.shields.io/badge/license-MIT-0F6E56)](LICENSE.txt) +[![Dependencies](https://img.shields.io/badge/dependencies-zero-085041)](smo_scottish_lidar.gemspec) + +A pure Ruby gem for listing and downloading Scottish Public Sector LiDAR data from the [Registry of Open Data on AWS](https://registry.opendata.aws/scottish-lidar/). + +Developed by **Sebastian Madrid Ontiveros** to support the hydraulic modelling community in Scotland. Whether you are building 1D-2D flood inundation models, defining subcatchments, or preparing terrain inputs for InfoWorks ICM, this gem takes the friction out of getting LiDAR data onto your machine. + +No AWS CLI. No credentials. No dependencies. Uses only Ruby stdlib (`net/http`, `uri`, `fileutils`). Compatible with the InfoWorks ICM embedded Ruby environment. + +

+ + Buy Me a Coffee QR code + +
+ buymeacoffee.com/smadrid +

+ +--- + +## What is this data? + +The Scottish Government publishes LiDAR survey data as open data through an S3 bucket (`srsp-open-data`). Coverage spans most of Scotland across five survey phases, plus a dedicated Outer Hebrides survey. Each phase includes three dataset types: + +| Type | Description | +|------|-------------| +| **DSM** | Digital Surface Model. Includes buildings, trees, and structures. Used for overland flow modelling. | +| **DTM** | Digital Terrain Model. Bare earth, vegetation removed. Ground surface for 2D mesh generation. | +| **LAZ** | Compressed LiDAR point cloud. Raw survey returns for processing in specialist software. | + +Files are organised by OS National Grid square (e.g. `NS`, `NT`, `NO`, `NN`) and are free to access. + +| Phase | Coverage | +|-------|----------| +| `phase-1` | Central Scotland | +| `phase-2` | South and East Scotland | +| `phase-3` | North and West Scotland | +| `phase-4` | Additional coverage | +| `phase-5` | Latest survey phase | +| `outer-hebrides` | Western Isles. Available at 25 cm and 50 cm resolution. | + +--- + +## Installation + +```sh +gem install smo_scottish_lidar +``` + +Or add to your Gemfile: + +```ruby +gem "smo_scottish_lidar" +``` + +--- + +## Quick start + +```ruby +require "smo_scottish_lidar" + +# See what Phase 1 DSM tiles are available in the NS grid square +lister = SmoScottishLidar::Lister.new +lister.summary("phase-1", "dsm", grid_square: "NS") + +# Download all Phase 1 DTM tiles for the NS grid square +downloader = SmoScottishLidar::Downloader.new +downloader.download("phase-1", "dtm", + destination: "/projects/my_catchment/lidar/dtm", + grid_square: "NS" +) + +# Download a single known tile +downloader.download_file("phase-1", "dsm", "NS56_1M_DSM_PHASE1.tif", + destination: "/tmp/lidar" +) +``` + +--- + +## Typical hydraulic modelling workflow + +The pattern below covers the full cycle from discovery to download, including safe resume if the connection drops. + +```ruby +require "smo_scottish_lidar" + +lister = SmoScottishLidar::Lister.new +downloader = SmoScottishLidar::Downloader.new(verbose: true) + +# Step 1. Check what is available for your catchment +lister.summary("phase-1", "dtm", grid_square: "NS") + +# Step 2. Dry run to confirm file count and total size before committing +downloader.download("phase-1", "dtm", + destination: "/projects/my_catchment/lidar/dtm", + grid_square: "NS", + dry_run: true +) + +# Step 3. Download for real +downloader.download("phase-1", "dtm", + destination: "/projects/my_catchment/lidar/dtm", + grid_square: "NS" +) + +# Step 4. If the download is interrupted, re-run step 3. +# Files already on disk at the correct size are skipped automatically. +``` + +--- + +## API reference + +### `SmoScottishLidar::Lister` + +Lists available files from the S3 bucket. Filtering by grid square is applied client-side after the full listing is fetched. + +```ruby +lister = SmoScottishLidar::Lister.new(verbose: false) +``` + +#### `lister.list(phase, type, grid_square: nil, resolution: nil)` + +Returns an `Array` of matching files. + +| Key | Type | Description | +|-----|------|-------------| +| `:key` | String | Full S3 object key | +| `:filename` | String | Bare filename, e.g. `NS56_1M_DSM_PHASE1.tif` | +| `:size` | Integer | File size in bytes | +| `:last_modified` | String | ISO 8601 timestamp | + +```ruby +files = lister.list("phase-1", "dsm", grid_square: "NS") +files.each { |f| puts "#{f[:filename]} (#{f[:size]} bytes)" } +``` + +#### `lister.summary(phase, type, grid_square: nil, resolution: nil)` + +Prints a formatted table to stdout and returns the same `Array`. + +```ruby +lister.summary("phase-2", "dtm", grid_square: "NT") +lister.summary("outer-hebrides", "dsm", resolution: "50cm") +``` + +--- + +### `SmoScottishLidar::Downloader` + +Downloads files from the S3 bucket. Responses are streamed in chunks so large files never load fully into memory. + +```ruby +downloader = SmoScottishLidar::Downloader.new(verbose: false) +``` + +#### `downloader.download(phase, type, destination:, **options)` + +Batch download with filtering. Returns `{ downloaded: [...], skipped: [...], failed: [...] }`. + +```ruby +downloader.download( + "phase-3", "dsm", + destination: "/data/lidar", + grid_square: "NO", + skip_existing: true, + dry_run: false +) +``` + +| Option | Default | Description | +|--------|---------|-------------| +| `destination:` | required | Local directory to write files into | +| `grid_square:` | `nil` | OS National Grid square filter, e.g. `"NS"`. `nil` downloads everything. | +| `resolution:` | `nil` | Outer Hebrides only. `"25cm"`, `"50cm"`, `"4ppm"`, or `"16ppm"`. | +| `skip_existing:` | `true` | Skip any file already on disk whose size matches S3. | +| `dry_run:` | `false` | Print the download plan without transferring anything. | + +#### `downloader.download_file(phase, type, filename, destination:, resolution: nil)` + +Downloads a single tile by exact filename. + +```ruby +downloader.download_file( + "phase-1", "dsm", "NS56_1M_DSM_PHASE1.tif", + destination: "/tmp/lidar" +) +``` + +--- + +### `SmoScottishLidar.prefix_for(phase, type, resolution: nil)` + +Returns the S3 prefix string for a given phase and type. Useful for building custom queries against the bucket. + +```ruby +SmoScottishLidar.prefix_for("phase-1", "dsm") +# => "lidar/phase-1/dsm/27700/gridded/" + +SmoScottishLidar.prefix_for("outer-hebrides", "dtm", resolution: "50cm") +# => "lidar/outer-hebrides/2019/dtm/50cm/27700/gridded/" +``` + +--- + +## Valid parameters + +```ruby +SmoScottishLidar::PHASES +# => ["phase-1", "phase-2", "phase-3", "phase-4", "phase-5", "outer-hebrides"] + +SmoScottishLidar::DATASET_TYPES +# => ["dsm", "dtm", "laz"] +``` + +### Outer Hebrides resolutions + +| Type | Available | Default | +|------|-----------|---------| +| `dsm` | `"25cm"`, `"50cm"` | `"25cm"` | +| `dtm` | `"25cm"`, `"50cm"` | `"25cm"` | +| `laz` | `"4ppm"`, `"16ppm"` | `"4ppm"` | + +Phases 1-5 do not take a `resolution:` argument. + +--- + +## Example scripts + +The `examples/` directory contains ready-to-run scripts covering every phase, type, and common use case. + +| Script | What it does | +|--------|-------------| +| `demo.rb` | Full walkthrough of all gem features | +| `phase_1_dsm.rb` | List Phase 1 DSM tiles | +| `phase_1_dtm.rb` | List Phase 1 DTM tiles | +| `phase_1_laz.rb` | List Phase 1 LAZ tiles | +| `phase_2_dsm.rb` ... | One script per phase and type | +| `outer_hebrides_dsm.rb` | Outer Hebrides DSM at both resolutions | +| `outer_hebrides_dtm.rb` | Outer Hebrides DTM at both resolutions | +| `outer_hebrides_laz.rb` | Outer Hebrides LAZ at both densities | +| `download_individual_tile.rb` | Download a single named tile | +| `download_batch_tiles.rb` | Batch download with grid square filter | + +```sh +ruby examples/demo.rb +ruby examples/phase_1_dsm.rb +``` + +--- + +## Data source + +Scottish Public Sector LiDAR Dataset, Scottish Government. +Available via [Registry of Open Data on AWS](https://registry.opendata.aws/scottish-lidar/) and the [Scottish Remote Sensing Portal](https://remotesensingdata.gov.scot). +S3 bucket: `s3://srsp-open-data/lidar/` +Access: public, no AWS account or credentials required. + +--- + +## Support + +If this gem saves you time on a project, consider buying me a coffee. + +

+ + Buy Me a Coffee QR code + +
+ buymeacoffee.com/smadrid +

+ +--- + +## License + +MIT. Copyright (c) 2025 Sebastian Madrid Ontiveros. diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/Rakefile b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/Rakefile new file mode 100644 index 00000000..6066e1a1 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/Rakefile @@ -0,0 +1,9 @@ +require "rake/testtask" + +Rake::TestTask.new(:test) do |t| + t.libs << "spec" + t.libs << "lib" + t.test_files = FileList["spec/**/*_spec.rb"] +end + +task default: :test diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/demo.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/demo.rb new file mode 100644 index 00000000..a43ffd59 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/demo.rb @@ -0,0 +1,94 @@ +require "smo_scottish_lidar" + +# ----------------------------------------------------------------------------- +# 1. List Phase 1 DSM files for grid square NS +# ----------------------------------------------------------------------------- +puts "=" * 60 +puts "1. PHASE 1 DSM - GRID SQUARE NS" +puts "=" * 60 + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-5", "dsm", grid_square: "NS") +puts + +# ----------------------------------------------------------------------------- +# 2. List Phase 1 DTM files for grid square NT +# ----------------------------------------------------------------------------- +puts "=" * 60 +puts "2. PHASE 1 DTM - GRID SQUARE NT" +puts "=" * 60 + +lister.summary("phase-5", "dtm", grid_square: "NT") +puts + +# ----------------------------------------------------------------------------- +# 3. List all LAZ files for Phase 2 +# ----------------------------------------------------------------------------- +puts "=" * 60 +puts "3. PHASE 2 LAZ - ALL GRID SQUARES" +puts "=" * 60 + +files = lister.list("phase-5", "laz") +puts "Total LAZ files in Phase 2: #{files.size}" +puts "First 5:" +files.first(5).each { |f| puts " #{f[:filename]} (#{f[:size]} bytes)" } +puts + +# ----------------------------------------------------------------------------- +# 4. Outer Hebrides DSM - default resolution (25cm) +# ----------------------------------------------------------------------------- +puts "=" * 60 +puts "4. OUTER HEBRIDES DSM - 25cm (default)" +puts "=" * 60 + +lister.summary("outer-hebrides", "dsm") +puts + +# ----------------------------------------------------------------------------- +# 5. Outer Hebrides DTM - 50cm resolution +# ----------------------------------------------------------------------------- +puts "=" * 60 +puts "5. OUTER HEBRIDES DTM - 50cm" +puts "=" * 60 + +lister.summary("outer-hebrides", "dtm", resolution: "50cm") +puts + +# ----------------------------------------------------------------------------- +# 6. Dry run download - Phase 1 DSM for NS56 +# ----------------------------------------------------------------------------- +puts "=" * 60 +puts "6. DRY RUN DOWNLOAD - PHASE 5 DSM NS56" +puts "=" * 60 + +downloader = SmoScottishLidar::Downloader.new +downloader.download( + "phase-5", "dsm", + destination: "/tmp/lidar_test", + grid_square: "NS96", + dry_run: true +) +puts + +# ----------------------------------------------------------------------------- +# 7. Prefix builder - all phases and types +# ----------------------------------------------------------------------------- +puts "=" * 60 +puts "7. PREFIX BUILDER" +puts "=" * 60 + +%w[phase-1 phase-2 phase-3 phase-4 phase-5].each do |phase| + %w[dsm dtm laz].each do |type| + puts " #{phase} / #{type}: #{SmoScottishLidar.prefix_for(phase, type)}" + end +end +puts +puts " outer-hebrides / dsm (25cm): #{SmoScottishLidar.prefix_for('outer-hebrides', 'dsm', resolution: '25cm')}" +puts " outer-hebrides / dsm (50cm): #{SmoScottishLidar.prefix_for('outer-hebrides', 'dsm', resolution: '50cm')}" +puts " outer-hebrides / laz (4ppm): #{SmoScottishLidar.prefix_for('outer-hebrides', 'laz', resolution: '4ppm')}" +puts " outer-hebrides / laz (16ppm): #{SmoScottishLidar.prefix_for('outer-hebrides', 'laz', resolution: '16ppm')}" +puts + +puts "=" * 60 +puts "All done." +puts "=" * 60 diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/download_batch_tiles.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/download_batch_tiles.rb new file mode 100644 index 00000000..37775f75 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/download_batch_tiles.rb @@ -0,0 +1,50 @@ +require "smo_scottish_lidar" + +# Batch download tiles with flexible filtering. +# Adjust the options below to match what you need. + +# --- Options --- +PHASE = "phase-1" +TYPE = "dsm" +DESTINATION = "/tmp/lidar/batch" + +# Filter by OS grid square prefix. Set to nil to download everything. +# Examples: "NS" (all NS tiles), "NT", "NO", nil (all tiles in phase) +GRID_SQUARE = "NS" + +# Set to true to see what would be downloaded without actually downloading. +DRY_RUN = false + +# Set to false to re-download files that already exist locally. +SKIP_EXISTING = true +# --------------- + +downloader = SmoScottishLidar::Downloader.new(verbose: false) +results = downloader.download( + PHASE, TYPE, + destination: DESTINATION, + grid_square: GRID_SQUARE, + skip_existing: SKIP_EXISTING, + dry_run: DRY_RUN +) + +puts +puts "Downloaded : #{results[:downloaded].size} file(s)" +puts "Skipped : #{results[:skipped].size} file(s)" +puts "Failed : #{results[:failed].size} file(s)" + +# ---- Batch download multiple specific filenames ---- +# If you already know the exact tiles you want, list them here: + +TILES = %w[ + NS56_1M_DSM_PHASE1.tif + NS57_1M_DSM_PHASE1.tif + NS58_1M_DSM_PHASE1.tif +] + +# Uncomment the block below to download only those specific tiles: +# puts +# puts "Downloading specific tiles..." +# TILES.each do |filename| +# downloader.download_file(PHASE, TYPE, filename, destination: DESTINATION) +# end diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/download_individual_tile.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/download_individual_tile.rb new file mode 100644 index 00000000..334e4b93 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/download_individual_tile.rb @@ -0,0 +1,11 @@ +require "smo_scottish_lidar" + +# Download a single specific tile by filename. +# Change these three values to what you need. +PHASE = "phase-1" +TYPE = "dsm" +FILENAME = "NS56_1M_DSM_PHASE1.tif" +DESTINATION = "/tmp/lidar/single" + +downloader = SmoScottishLidar::Downloader.new(verbose: true) +downloader.download_file(PHASE, TYPE, FILENAME, destination: DESTINATION) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_dsm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_dsm.rb new file mode 100644 index 00000000..f3731e9c --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_dsm.rb @@ -0,0 +1,18 @@ +require "smo_scottish_lidar" + +# outer-hebrides DSM - Digital Surface Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +# resolution: "25cm", "50cm" for dsm/dtm. "4ppm", "16ppm" for laz. +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("outer-hebrides", "dsm", grid_square: GRID_SQUARE, resolution: "25cm") + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("outer-hebrides", "dsm", +# destination: "/tmp/lidar/outer-hebrides/dsm", +# resolution: "25cm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_dtm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_dtm.rb new file mode 100644 index 00000000..be948d67 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_dtm.rb @@ -0,0 +1,18 @@ +require "smo_scottish_lidar" + +# outer-hebrides DTM - Digital Terrain Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +# resolution: "25cm", "50cm" for dsm/dtm. "4ppm", "16ppm" for laz. +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("outer-hebrides", "dtm", grid_square: GRID_SQUARE, resolution: "25cm") + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("outer-hebrides", "dtm", +# destination: "/tmp/lidar/outer-hebrides/dtm", +# resolution: "25cm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_laz.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_laz.rb new file mode 100644 index 00000000..4039e4e8 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/outer_hebrides_laz.rb @@ -0,0 +1,18 @@ +require "smo_scottish_lidar" + +# outer-hebrides LAZ - LiDAR Point Cloud (LAZ) +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +# resolution: "25cm", "50cm" for dsm/dtm. "4ppm", "16ppm" for laz. +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("outer-hebrides", "laz", grid_square: GRID_SQUARE, resolution: "25cm") + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("outer-hebrides", "laz", +# destination: "/tmp/lidar/outer-hebrides/laz", +# resolution: "25cm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_dsm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_dsm.rb new file mode 100644 index 00000000..4d514fc3 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_dsm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# Phase 1 DSM - Digital Surface Model +# Grid square filter: change to nil to list all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-1", "dsm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-1", "dsm", +# destination: "/tmp/lidar/phase-1/dsm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_dtm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_dtm.rb new file mode 100644 index 00000000..e7259f00 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_dtm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-1 DTM - Digital Terrain Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-1", "dtm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-1", "dtm", +# destination: "/tmp/lidar/phase-1/dtm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_laz.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_laz.rb new file mode 100644 index 00000000..88b95360 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_1_laz.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-1 LAZ - LiDAR Point Cloud (LAZ) +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-1", "laz", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-1", "laz", +# destination: "/tmp/lidar/phase-1/laz", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_dsm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_dsm.rb new file mode 100644 index 00000000..2da142ee --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_dsm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-2 DSM - Digital Surface Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-2", "dsm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-2", "dsm", +# destination: "/tmp/lidar/phase-2/dsm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_dtm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_dtm.rb new file mode 100644 index 00000000..fa4a769f --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_dtm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-2 DTM - Digital Terrain Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-2", "dtm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-2", "dtm", +# destination: "/tmp/lidar/phase-2/dtm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_laz.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_laz.rb new file mode 100644 index 00000000..5d9cc1bc --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_2_laz.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-2 LAZ - LiDAR Point Cloud (LAZ) +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-2", "laz", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-2", "laz", +# destination: "/tmp/lidar/phase-2/laz", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_dsm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_dsm.rb new file mode 100644 index 00000000..dc0697fc --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_dsm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-3 DSM - Digital Surface Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-3", "dsm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-3", "dsm", +# destination: "/tmp/lidar/phase-3/dsm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_dtm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_dtm.rb new file mode 100644 index 00000000..06c84e79 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_dtm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-3 DTM - Digital Terrain Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-3", "dtm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-3", "dtm", +# destination: "/tmp/lidar/phase-3/dtm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_laz.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_laz.rb new file mode 100644 index 00000000..18c75d00 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_3_laz.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-3 LAZ - LiDAR Point Cloud (LAZ) +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-3", "laz", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-3", "laz", +# destination: "/tmp/lidar/phase-3/laz", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_dsm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_dsm.rb new file mode 100644 index 00000000..6dada7be --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_dsm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-4 DSM - Digital Surface Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-4", "dsm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-4", "dsm", +# destination: "/tmp/lidar/phase-4/dsm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_dtm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_dtm.rb new file mode 100644 index 00000000..06394329 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_dtm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-4 DTM - Digital Terrain Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-4", "dtm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-4", "dtm", +# destination: "/tmp/lidar/phase-4/dtm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_laz.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_laz.rb new file mode 100644 index 00000000..e6a4011c --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_4_laz.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-4 LAZ - LiDAR Point Cloud (LAZ) +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-4", "laz", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-4", "laz", +# destination: "/tmp/lidar/phase-4/laz", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_dsm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_dsm.rb new file mode 100644 index 00000000..eaeb96c9 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_dsm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-5 DSM - Digital Surface Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-5", "dsm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-5", "dsm", +# destination: "/tmp/lidar/phase-5/dsm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_dtm.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_dtm.rb new file mode 100644 index 00000000..9eced10a --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_dtm.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-5 DTM - Digital Terrain Model +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-5", "dtm", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-5", "dtm", +# destination: "/tmp/lidar/phase-5/dtm", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_laz.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_laz.rb new file mode 100644 index 00000000..df7d45df --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/examples/phase_5_laz.rb @@ -0,0 +1,16 @@ +require "smo_scottish_lidar" + +# phase-5 LAZ - LiDAR Point Cloud (LAZ) +# Grid square filter: set to nil for all, or e.g. "NS", "NT", "NO" +GRID_SQUARE = nil + +lister = SmoScottishLidar::Lister.new +lister.summary("phase-5", "laz", grid_square: GRID_SQUARE) + +# Uncomment to download: +# downloader = SmoScottishLidar::Downloader.new +# downloader.download("phase-5", "laz", +# destination: "/tmp/lidar/phase-5/laz", +# grid_square: GRID_SQUARE, +# dry_run: false +# ) diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar.rb new file mode 100644 index 00000000..7909d197 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require_relative "smo_scottish_lidar/version" +require_relative "smo_scottish_lidar/constants" +require_relative "smo_scottish_lidar/client" +require_relative "smo_scottish_lidar/downloader" +require_relative "smo_scottish_lidar/lister" diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/client.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/client.rb new file mode 100644 index 00000000..21042794 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/client.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +require "net/http" +require "uri" + +module SmoScottishLidar + # Low-level S3 client. Uses only Ruby stdlib net/http. + # Accesses the public (unsigned) srsp-open-data bucket directly over HTTPS. + class Client + MAX_KEYS = 1000 + MSG_TOO_MANY_REDIRECTS = "Too many redirects" + MSG_NO_LOCATION_HEADER = "Redirect with no Location header" + + def initialize(verbose: false) + @verbose = verbose + end + + # Lists all object keys under a given S3 prefix. + # Handles S3 pagination (continuation tokens) automatically. + # Returns an Array of Hashes: [{ key:, size:, last_modified: }, ...] + def list_objects(prefix) + objects = [] + continuation_token = nil + + loop do + xml = fetch_list_page(prefix, continuation_token) + page_objects, next_token, truncated = parse_list_response(xml) + objects.concat(page_objects) + + break unless truncated + + continuation_token = next_token + end + + objects + end + + # Downloads a single S3 object key to a local file path. + # Follows redirects, streams in chunks to avoid loading into memory. + # Returns true on success, raises on error. + def download_object(key, local_path, &progress_block) + url = "#{BASE_URL}/#{key}" + log "Downloading: #{url} -> #{local_path}" + + uri = URI.parse(url) + fetch_with_redirect(uri, local_path, &progress_block) + true + end + + private + + def fetch_list_page(prefix, continuation_token) + params = { + "list-type" => "2", + "prefix" => prefix, + "max-keys" => MAX_KEYS.to_s + } + params["continuation-token"] = continuation_token if continuation_token + + query = params.map { |k, v| "#{uri_encode(k)}=#{uri_encode(v)}" }.join("&") + uri = URI.parse("#{BASE_URL}/?#{query}") + + log "Listing: #{uri}" + response = get_response(uri) + response.body + end + + def parse_list_response(xml) + objects = [] + + xml.scan(%r{(.*?)}m).each do |match| + block = match[0] + key = extract_tag(block, "Key") + size = extract_tag(block, "Size").to_i + mtime = extract_tag(block, "LastModified") + objects << { key: key, size: size, last_modified: mtime } + end + + truncated = extract_tag(xml, "IsTruncated") == "true" + next_token = extract_tag(xml, "NextContinuationToken") + + [objects, next_token, truncated] + end + + def fetch_with_redirect(uri, local_path, redirects_remaining: 5, &progress_block) + raise MSG_TOO_MANY_REDIRECTS if redirects_remaining.zero? + + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http| + request = Net::HTTP::Get.new(uri.request_uri) + http.request(request) do |response| + case response.code.to_i + when 200 + total = response["content-length"]&.to_i + received = 0 + File.open(local_path, "wb") do |f| + response.read_body do |chunk| + f.write(chunk) + received += chunk.bytesize + progress_block&.call(received, total) + end + end + when 301, 302, 307, 308 + location = response["location"] + raise MSG_NO_LOCATION_HEADER unless location + + log "Redirect -> #{location}" + fetch_with_redirect(URI.parse(location), local_path, + redirects_remaining: redirects_remaining - 1, + &progress_block) + else + raise "HTTP #{response.code} for #{uri}" + end + end + end + end + + def get_response(uri, redirects_remaining: 5) + raise "#{MSG_TOO_MANY_REDIRECTS} listing #{uri}" if redirects_remaining.zero? + + Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http| + response = http.get(uri.request_uri) + case response.code.to_i + when 200 + response + when 301, 302, 307, 308 + location = response["location"] + raise MSG_NO_LOCATION_HEADER unless location + + log "Redirect -> #{location}" + get_response(URI.parse(location), redirects_remaining: redirects_remaining - 1) + else + raise "HTTP #{response.code} listing #{uri}" + end + end + end + + def extract_tag(xml, tag) + match = xml.match(%r{<#{tag}>(.*?)}m) + match ? match[1].strip : "" + end + + def uri_encode(str) + URI.encode_uri_component(str.to_s) + end + + def log(msg) + warn "[smo_scottish_lidar] #{msg}" if @verbose + end + end +end diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/constants.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/constants.rb new file mode 100644 index 00000000..cdbd45e2 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/constants.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module SmoScottishLidar + BUCKET = "srsp-open-data" + BASE_PREFIX = "lidar" + REGION = "eu-west-2" + BASE_URL = "https://#{BUCKET}.s3.#{REGION}.amazonaws.com" + + # Valid phases and dataset types derived from the Scottish Government + # Registry of Open Data on AWS documentation. + PHASES = %w[phase-1 phase-2 phase-3 phase-4 phase-5 outer-hebrides].freeze + DATASET_TYPES = %w[dsm dtm laz].freeze + + # Outer Hebrides has resolution sub-folders; all other phases do not. + OUTER_HEBRIDES_RESOLUTIONS = { + "dsm" => %w[25cm 50cm], + "dtm" => %w[25cm 50cm], + "laz" => %w[4ppm 16ppm] + }.freeze + + # Canonical S3 prefix builder. Returns the prefix string (no bucket). + # Examples: + # prefix_for("phase-1", "dsm") => "lidar/phase-1/dsm/27700/gridded/" + # prefix_for("outer-hebrides", "dtm", resolution: "50cm") => "lidar/outer-hebrides/2019/dtm/50cm/27700/gridded/" + def self.prefix_for(phase, type, resolution: nil) + validate_phase!(phase) + validate_type!(type) + + if phase == "outer-hebrides" + res = resolve_outer_hebrides_resolution(type, resolution) + "#{BASE_PREFIX}/outer-hebrides/2019/#{type}/#{res}/27700/gridded/" + else + "#{BASE_PREFIX}/#{phase}/#{type}/27700/gridded/" + end + end + + def self.validate_phase!(phase) + return if PHASES.include?(phase) + + raise ArgumentError, "Unknown phase '#{phase}'. Valid: #{PHASES.join(', ')}" + end + + def self.validate_type!(type) + return if DATASET_TYPES.include?(type) + + raise ArgumentError, "Unknown dataset type '#{type}'. Valid: #{DATASET_TYPES.join(', ')}" + end + + def self.resolve_outer_hebrides_resolution(type, resolution) + available = OUTER_HEBRIDES_RESOLUTIONS[type] + if resolution.nil? + available.first + elsif available.include?(resolution) + resolution + else + raise ArgumentError, + "Resolution '#{resolution}' not available for outer-hebrides #{type}. " \ + "Available: #{available.join(', ')}" + end + end +end diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/downloader.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/downloader.rb new file mode 100644 index 00000000..a696a804 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/downloader.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +require "fileutils" + +module SmoScottishLidar + # Downloads LiDAR files from the Scottish Government S3 bucket. + # No external dependencies. Uses only Ruby stdlib. + class Downloader + attr_reader :client, :lister + + def initialize(verbose: false) + @client = Client.new(verbose: verbose) + @lister = Lister.new(verbose: verbose) + @verbose = verbose + end + + # Download all files for a given phase/type, with optional filtering. + # + # @param phase [String] e.g. "phase-1", "outer-hebrides" + # @param type [String] "dsm", "dtm", or "laz" + # @param destination [String] Local directory to save files into + # @param grid_square [String, nil] OS grid square filter e.g. "NS" + # @param resolution [String, nil] Outer Hebrides resolution e.g. "50cm" + # @param skip_existing [Boolean] Skip files that already exist locally (default: true) + # @param dry_run [Boolean] List what would be downloaded without downloading + # @return [Hash] { downloaded: [...], skipped: [...], failed: [...] } + def download(phase, type, + destination:, + grid_square: nil, + resolution: nil, + skip_existing: true, + dry_run: false) + + FileUtils.mkdir_p(destination) unless dry_run + + objects = lister.list(phase, type, grid_square: grid_square, resolution: resolution) + + if objects.empty? + puts "No files matched your criteria." + return { downloaded: [], skipped: [], failed: [] } + end + + total_bytes = objects.sum { |o| o[:size] } + puts "Found #{objects.size} file(s) (#{format_bytes(total_bytes)} total)" + puts "Destination: #{destination}" + puts "(Dry run - no files will be downloaded)" if dry_run + puts + + results = { downloaded: [], skipped: [], failed: [] } + + objects.each_with_index do |obj, idx| + local_path = File.join(destination, obj[:filename]) + label = "[#{idx + 1}/#{objects.size}] #{obj[:filename]} (#{format_bytes(obj[:size])})" + + if skip_existing && File.exist?(local_path) && File.size(local_path) == obj[:size] + puts "SKIP #{label}" + results[:skipped] << obj[:filename] + next + end + + if dry_run + puts "WOULD #{label}" + results[:downloaded] << obj[:filename] + next + end + + print "GET #{label} ... " + $stdout.flush + + begin + client.download_object(obj[:key], local_path) do |received, total| + next unless @verbose && total + + pct = (received.to_f / total * 100).round(1) + print "\rGET #{label} ... #{pct}%" + $stdout.flush + end + puts "OK" + results[:downloaded] << obj[:filename] + rescue StandardError => e + puts "FAILED (#{e.message})" + results[:failed] << { file: obj[:filename], error: e.message } + end + end + + print_summary(results) + results + end + + # Convenience: download a single file by its S3 key or filename. + # + # @param phase [String] + # @param type [String] + # @param filename [String] Exact filename to download + # @param destination [String] Local directory + # @param resolution [String, nil] + def download_file(phase, type, filename, destination:, resolution: nil) + prefix = SmoScottishLidar.prefix_for(phase, type, resolution: resolution) + key = "#{prefix}#{filename}" + local = File.join(destination, filename) + + FileUtils.mkdir_p(destination) + puts "Downloading #{filename} ..." + client.download_object(key, local) + puts "Saved to #{local}" + local + end + + private + + def print_summary(results) + puts + puts "Done. Downloaded: #{results[:downloaded].size}, " \ + "Skipped: #{results[:skipped].size}, " \ + "Failed: #{results[:failed].size}" + + return if results[:failed].empty? + + puts "Failed files:" + results[:failed].each { |f| puts " #{f[:file]}: #{f[:error]}" } + end + + def format_bytes(bytes) + units = %w[B KB MB GB TB] + idx = 0 + size = bytes.to_f + while size >= 1024 && idx < units.size - 1 + size /= 1024.0 + idx += 1 + end + format("%.1f %s", size, units[idx]) + end + end +end diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/lister.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/lister.rb new file mode 100644 index 00000000..a291a662 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/lister.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module SmoScottishLidar + # Lists available LiDAR files from the Scottish Government S3 bucket. + # All filtering is done client-side after fetching the S3 listing. + class Lister + attr_reader :client + + def initialize(verbose: false) + @client = Client.new(verbose: verbose) + end + + # List files for a given phase and dataset type. + # + # @param phase [String] e.g. "phase-1", "outer-hebrides" + # @param type [String] "dsm", "dtm", or "laz" + # @param grid_square [String, nil] Optional OS grid square filter, e.g. "NS", "NT" + # @param resolution [String, nil] For outer-hebrides only, e.g. "50cm", "4ppm" + # @return [Array] Array of { key:, size:, last_modified:, filename: } + def list(phase, type, grid_square: nil, resolution: nil) + prefix = SmoScottishLidar.prefix_for(phase, type, resolution: resolution) + objects = client.list_objects(prefix) + + objects.map! do |obj| + obj.merge(filename: File.basename(obj[:key])) + end + + if grid_square + pattern = grid_square.upcase + objects.select! { |obj| obj[:filename].upcase.start_with?(pattern) } + end + + objects + end + + # Print a human-readable summary of available files. + def summary(phase, type, grid_square: nil, resolution: nil) + objects = list(phase, type, grid_square: grid_square, resolution: resolution) + + if objects.empty? + puts "No files found." + return objects + end + + total_bytes = objects.sum { |o| o[:size] } + + puts "Phase : #{phase}" + puts "Type : #{type}" + puts "Grid sq. : #{grid_square || '(all)'}" + puts "Files : #{objects.size}" + puts "Total size: #{format_bytes(total_bytes)}" + puts + puts format("%-50s %10s %s", "Filename", "Size", "Last Modified") + puts "-" * 80 + objects.each do |obj| + puts format("%-50s %10s %s", obj[:filename], format_bytes(obj[:size]), obj[:last_modified]) + end + + objects + end + + private + + def format_bytes(bytes) + units = %w[B KB MB GB TB] + idx = 0 + size = bytes.to_f + while size >= 1024 && idx < units.size - 1 + size /= 1024.0 + idx += 1 + end + format("%.1f %s", size, units[idx]) + end + end +end diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/version.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/version.rb new file mode 100644 index 00000000..0c897aa8 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/lib/smo_scottish_lidar/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module SmoScottishLidar + VERSION = "0.1.1" +end diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/smo_scottish_lidar.gemspec b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/smo_scottish_lidar.gemspec new file mode 100644 index 00000000..a92f0710 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/smo_scottish_lidar.gemspec @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require_relative "lib/smo_scottish_lidar/version" + +Gem::Specification.new do |spec| + spec.name = "smo_scottish_lidar" + spec.version = SmoScottishLidar::VERSION + spec.authors = ["Sebastian Madrid Ontiveros"] + spec.email = ["sebasmadrid20@hotmail.com"] + + spec.summary = "Download Scottish Public Sector LiDAR data from the Registry of Open Data on AWS." + spec.description = <<~DESC + Developed by Sebastian Madrid Ontiveros to support hydraulic modellers in Scotland + building 1D-2D hydraulic models and flood risk assessments. Provides a pure Ruby + interface for listing and downloading Scottish Public Sector LiDAR datasets (DSM, + DTM, LAZ) from the Registry of Open Data on AWS. Supports all survey phases (1-5) + and Outer Hebrides, OS National Grid square filtering, paginated S3 listing, streamed + downloads with resume support, and dry-run mode. No external dependencies. Uses only + Ruby stdlib (net/http, uri, fileutils). If this gem saves you time, consider buying + Sebastian a coffee at https://buymeacoffee.com/smadrid + DESC + spec.homepage = "https://github.com/Sebasmadridmx/smo_scottish_lidar" + spec.license = "MIT" + spec.required_ruby_version = ">= 2.7.0" + + spec.files = Dir["lib/**/*.rb", "README.md", "LICENSE", "CHANGELOG.md"] + spec.require_paths = ["lib"] + + # No runtime dependencies. Uses only Ruby stdlib: net/http, uri, fileutils. +end diff --git a/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/spec/smo_scottish_lidar_spec.rb b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/spec/smo_scottish_lidar_spec.rb new file mode 100644 index 00000000..5ca8f148 --- /dev/null +++ b/07 Ruby Exchange Gems/SMO-Scottish-LiDAR/spec/smo_scottish_lidar_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require_relative "../lib/smo_scottish_lidar" + +# Minimal inline tests - no test framework dependency. +# Run with: ruby spec/smo_scottish_lidar_spec.rb + +errors = [] + +def assert(description, &block) + condition = block.call + if condition + puts "PASS #{description}" + else + puts "FAIL #{description}" + $errors_count = ($errors_count || 0) + 1 + end +end + +def assert_raises(description, &block) + block.call + puts "FAIL #{description} (no exception raised)" + $errors_count = ($errors_count || 0) + 1 +rescue ArgumentError + puts "PASS #{description}" +end + +puts "=== smo_scottish_lidar unit tests ===" +puts + +# --- Constants / prefix builder --- + +assert "phase-1 DSM prefix" do + SmoScottishLidar.prefix_for("phase-1", "dsm") == "lidar/phase-1/dsm/27700/gridded/" +end + +assert "phase-5 LAZ prefix" do + SmoScottishLidar.prefix_for("phase-5", "laz") == "lidar/phase-5/laz/27700/gridded/" +end + +assert "outer-hebrides DTM 50cm prefix" do + SmoScottishLidar.prefix_for("outer-hebrides", "dtm", resolution: "50cm") == + "lidar/outer-hebrides/2019/dtm/50cm/27700/gridded/" +end + +assert "outer-hebrides DSM defaults to 25cm" do + SmoScottishLidar.prefix_for("outer-hebrides", "dsm") == + "lidar/outer-hebrides/2019/dsm/25cm/27700/gridded/" +end + +assert "outer-hebrides LAZ defaults to 4ppm" do + SmoScottishLidar.prefix_for("outer-hebrides", "laz") == + "lidar/outer-hebrides/2019/laz/4ppm/27700/gridded/" +end + +assert_raises "invalid phase raises ArgumentError" do + SmoScottishLidar.prefix_for("phase-99", "dsm") +end + +assert_raises "invalid type raises ArgumentError" do + SmoScottishLidar.prefix_for("phase-1", "xyz") +end + +assert_raises "invalid outer-hebrides resolution raises ArgumentError" do + SmoScottishLidar.prefix_for("outer-hebrides", "dtm", resolution: "1m") +end + +# --- Version --- +assert "version is a string" do + SmoScottishLidar::VERSION.is_a?(String) +end + +assert "version format x.y.z" do + SmoScottishLidar::VERSION.match?(/\A\d+\.\d+\.\d+\z/) +end + +# --- Client instantiation --- +assert "Client instantiates without error" do + SmoScottishLidar::Client.new + true +end + +assert "Lister instantiates without error" do + SmoScottishLidar::Lister.new + true +end + +assert "Downloader instantiates without error" do + SmoScottishLidar::Downloader.new + true +end + +puts +errors_count = $errors_count || 0 +if errors_count.zero? + puts "All tests passed." +else + puts "#{errors_count} test(s) failed." + exit 1 +end