From da1e05bdff19ebb926cda7d9f85507144baba45f Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 12:32:07 +0000 Subject: [PATCH 01/21] feat: add edgenode-harvester container variant Introduces a new container variant (Dockerfile.harvester) that bundles log-harvester (unytco/log-harvester) instead of log-sender. The harvester reads from log-collector, aggregates usage data, and parks invoices on Unyt Agreements via an embedded Holochain conductor. Key changes: - Dockerfile.harvester: single-stage wolfi-base build, clones and builds log-harvester from source via GITHUB_TOKEN build secret, bakes in unyt.happ from latest unytco/unyt-sandbox release (pinnable via UNYT_HAPP_VERSION build arg), exposes ports 4444 and 4445 - s6-overlay-harvester/: self-contained s6 service tree (conductor, log-harvester, logrotate-cron, setup) with no dependency on the base s6-overlay directory - log-harvester s6 service: waits for conductor readiness, installs unyt.happ, attaches app websocket on 4445, init/refreshes harvester config, runs harvester in loop mode - CI: build-and-push-harvester-image job publishing ghcr.io/holo-host/edgenode-harvester using HARVESTER_REPO_TOKEN secret - docker-compose.yml: edgenode-harvester service for local testing - Docs: LOG_HARVESTER_QUICKSTART.md, updated README.md, docker/README.md, docker/CHANGELOG.md Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/release.yml | 40 +++++ README.md | 34 +++- docker/CHANGELOG.md | 8 + docker/Dockerfile.harvester | 106 ++++++++++++ docker/LOG_HARVESTER_QUICKSTART.md | 159 ++++++++++++++++++ docker/README.md | 24 ++- docker/docker-compose.yml | 38 +++++ .../s6-rc.d/conductor/dependencies.d/setup | 0 .../s6-rc.d/conductor/run | 6 + .../s6-rc.d/conductor/type | 1 + .../log-harvester/dependencies.d/setup | 0 .../s6-rc.d/log-harvester/run | 76 +++++++++ .../s6-rc.d/log-harvester/type | 1 + .../logrotate-cron/dependencies.d/setup | 0 .../s6-rc.d/logrotate-cron/run | 5 + .../s6-rc.d/logrotate-cron/type | 1 + .../s6-rc.d/setup/init.sh | 40 +++++ .../s6-rc.d/user/contents.d/conductor | 0 .../s6-rc.d/user/contents.d/log-harvester | 0 .../s6-rc.d/user/contents.d/logrotate-cron | 0 docker/s6-overlay-harvester/s6-rc.d/user/type | 1 + 21 files changed, 533 insertions(+), 7 deletions(-) create mode 100644 docker/Dockerfile.harvester create mode 100644 docker/LOG_HARVESTER_QUICKSTART.md create mode 100644 docker/s6-overlay-harvester/s6-rc.d/conductor/dependencies.d/setup create mode 100644 docker/s6-overlay-harvester/s6-rc.d/conductor/run create mode 100644 docker/s6-overlay-harvester/s6-rc.d/conductor/type create mode 100644 docker/s6-overlay-harvester/s6-rc.d/log-harvester/dependencies.d/setup create mode 100644 docker/s6-overlay-harvester/s6-rc.d/log-harvester/run create mode 100644 docker/s6-overlay-harvester/s6-rc.d/log-harvester/type create mode 100644 docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/dependencies.d/setup create mode 100644 docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/run create mode 100644 docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/type create mode 100644 docker/s6-overlay-harvester/s6-rc.d/setup/init.sh create mode 100644 docker/s6-overlay-harvester/s6-rc.d/user/contents.d/conductor create mode 100644 docker/s6-overlay-harvester/s6-rc.d/user/contents.d/log-harvester create mode 100644 docker/s6-overlay-harvester/s6-rc.d/user/contents.d/logrotate-cron create mode 100644 docker/s6-overlay-harvester/s6-rc.d/user/type diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f3b4182..5275a76 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -156,3 +156,43 @@ jobs: --push \ --tag "${VERSION_TAG}" \ --tag "${LATEST_TAG}" + + build-and-push-harvester-image: + runs-on: ubuntu-latest + needs: create-release + if: github.ref_type == 'tag' || github.event_name == 'workflow_dispatch' + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ needs.create-release.outputs.tag }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push harvester image + run: | + VERSION_TAG="ghcr.io/holo-host/edgenode-harvester:${{ needs.create-release.outputs.tag }}" + LATEST_TAG="ghcr.io/holo-host/edgenode-harvester:latest" + + docker buildx build docker/ --file docker/Dockerfile.harvester \ + --platform linux/amd64,linux/arm64 \ + --push \ + --tag "${VERSION_TAG}" \ + --tag "${LATEST_TAG}" \ + --secret id=github_token,env=GITHUB_TOKEN + env: + GITHUB_TOKEN: ${{ secrets.HARVESTER_REPO_TOKEN }} diff --git a/README.md b/README.md index 969f068..de18be9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repo contains the tooling needed to deploy and operate always-on nodes for The tooling consists of: -1. Edge Node - A Docker container specification for running Holochain with hApps in an OCI-compliant containerized environment. +1. Edge Node - Docker container specifications for running Holochain with hApps in an OCI-compliant containerized environment. Two variants: `edgenode` (standard, with log-sender) and `edgenode-harvester` (with log-harvester for Unyt invoice aggregation). 2. HolOS - A streamlined Linux ISO that enables the deployment of this container on physical or virtual hardware (especially HoloPorts). For a detailed overview and usage instructions [see here](/USAGE.md). @@ -23,8 +23,9 @@ For a detailed overview and usage instructions [see here](/USAGE.md). A [Docker-based container](docker/README.md) that delivers Edge Node, ready to run hApps: -- Holochain conductor configured to automatically run via `tini`. +- Holochain conductor managed by s6-overlay, starting automatically on container launch. - Tools for installing and managing hApps from configuration files. +- Two variants: standard (`edgenode`) and harvester (`edgenode-harvester`). ### HolOS Build System @@ -43,9 +44,9 @@ A [specialized OS builder](holos/README.md) for creating custom ISO images using ## Quick Start -### To test the Edge Node container: +### Standard Edge Node -1. Pull the Docker image: +1. Pull the image: ```sh docker pull ghcr.io/holo-host/edgenode @@ -64,6 +65,29 @@ docker exec -it edgenode su - nonroot ps -ef ``` +### Harvester Edge Node + +1. Pull the image: + +```sh +docker pull ghcr.io/holo-host/edgenode-harvester +``` + +2. Launch with your log-collector credentials: + +```sh +docker run --name harvester -dit \ + -v $(pwd)/holo-data:/data \ + -p 4444:4444 \ + -p 4445:4445 \ + -e COLLECTOR_URL=https://your-log-collector.unyt.dev \ + -e ADMIN_SECRET=your-admin-secret \ + -e LAIR_PASSWORD=your-lair-password \ + ghcr.io/holo-host/edgenode-harvester +``` + +See [docker/LOG_HARVESTER_QUICKSTART.md](docker/LOG_HARVESTER_QUICKSTART.md) for full setup instructions. + ### For HolOS Users 1. Download a release of the iso from our [releases page](https://github.com/Holo-Host/edgenode/releases) @@ -74,5 +98,7 @@ ps -ef - [Detailed overview and usage instructions](/USAGE.md) - [Edge Node Container Instructions](docker/README.md) +- [Log-Sender Quickstart (Unyt resource accounting)](docker/LOG_SENDER_QUICKSTART.md) +- [Log-Harvester Quickstart (Unyt invoice aggregation)](docker/LOG_HARVESTER_QUICKSTART.md) - [HolOS Build System Guide](holos/README.md) - [Tools for working with Edge Nodes](tools/README.md) diff --git a/docker/CHANGELOG.md b/docker/CHANGELOG.md index ff2afb6..1b79718 100644 --- a/docker/CHANGELOG.md +++ b/docker/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- `edgenode-harvester` container variant (`Dockerfile.harvester`) — bundles `log-harvester` (unytco/log-harvester) instead of `log-sender` for Unyt invoice aggregation +- `s6-overlay-harvester/` service tree: self-contained s6 services for the harvester variant (`conductor`, `log-harvester`, `logrotate-cron`, `setup`) +- `log-harvester` s6 longrun service: waits for conductor, installs `unyt.happ`, attaches app websocket on port 4445, initializes/refreshes harvester config, runs harvester loop +- `unyt.happ` baked into the harvester image from latest `unytco/unyt-sandbox` release; pinnable via `UNYT_HAPP_VERSION` build arg +- `LOG_HARVESTER_QUICKSTART.md` quickstart guide for the harvester variant +- CI `build-and-push-harvester-image` job in release workflow publishing `ghcr.io/holo-host/edgenode-harvester` + ## [0.1.0-alpha1] - 2026-03-13 ### Added diff --git a/docker/Dockerfile.harvester b/docker/Dockerfile.harvester new file mode 100644 index 0000000..a543e3f --- /dev/null +++ b/docker/Dockerfile.harvester @@ -0,0 +1,106 @@ +# syntax=docker/dockerfile:1 +# Dockerfile for the edge node harvester variant. +# Replaces log-sender with log-harvester (unytco/log-harvester). +# Requires GITHUB_TOKEN build secret to clone the private log-harvester repo: +# docker buildx build --secret id=github_token,env=GITHUB_TOKEN ... + +FROM cgr.dev/chainguard/wolfi-base + +VOLUME ["/data"] + +RUN apk update && apk upgrade && apk add --no-cache bash curl wget git htop jq strace tcpdump coreutils logrotate shadow gosu uuidgen xz libstdc++ nodejs npm + +ARG TARGETARCH +ARG S6_OVERLAY_VERSION=3.2.0.2 + +RUN case "${TARGETARCH}" in \ + amd64) S6_ARCH="x86_64" ;; \ + arm64) S6_ARCH="aarch64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + wget https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz && \ + wget https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-${S6_ARCH}.tar.xz && \ + tar -C / -Jxpf s6-overlay-noarch.tar.xz && \ + tar -C / -Jxpf s6-overlay-${S6_ARCH}.tar.xz && \ + rm -f s6-overlay-*.tar.xz + +ARG HOLOCHAIN_VERSION=0.6.1-rc.3 +ARG HC_VERSION=0.6.1-rc.3 + +RUN case "${TARGETARCH}" in \ + amd64) HC_ARCH="x86_64-unknown-linux-gnu" ;; \ + arm64) HC_ARCH="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + wget https://github.com/holochain/holochain/releases/download/holochain-${HOLOCHAIN_VERSION}/holochain-${HC_ARCH} && \ + wget https://github.com/holochain/holochain/releases/download/holochain-${HC_VERSION}/hc-${HC_ARCH} && \ + mv holochain-${HC_ARCH} /bin/holochain && \ + mv hc-${HC_ARCH} /bin/hc && \ + chmod +x /bin/holochain && \ + chmod +x /bin/hc + +ARG LOG_HARVESTER_VERSION=main + +RUN --mount=type=secret,id=github_token \ + mkdir -p /app && \ + git clone --depth 1 --branch ${LOG_HARVESTER_VERSION} \ + https://$(cat /run/secrets/github_token)@github.com/unytco/log-harvester.git /app/src && \ + cd /app/src && \ + npm ci && \ + npm run build && \ + cp -r dist /app/dist && \ + npm prune --omit=dev && \ + cp -r node_modules /app/node_modules && \ + cd / && rm -rf /app/src + +ARG UNYT_HAPP_VERSION=latest + +RUN if [ "${UNYT_HAPP_VERSION}" = "latest" ]; then \ + wget -O /app/unyt.happ \ + https://github.com/unytco/unyt-sandbox/releases/latest/download/unyt.happ; \ + else \ + wget -O /app/unyt.happ \ + https://github.com/unytco/unyt-sandbox/releases/download/${UNYT_HAPP_VERSION}/unyt.happ; \ + fi + +COPY happ_tool /usr/local/bin/happ_tool +RUN chmod +x /usr/local/bin/happ_tool +RUN ln -sf /usr/local/bin/happ_tool /usr/local/bin/install_happ && \ + ln -sf /usr/local/bin/happ_tool /usr/local/bin/uninstall_happ && \ + ln -sf /usr/local/bin/happ_tool /usr/local/bin/enable_happ && \ + ln -sf /usr/local/bin/happ_tool /usr/local/bin/disable_happ && \ + ln -sf /usr/local/bin/happ_tool /usr/local/bin/list_happs && \ + chmod +x /usr/local/bin/install_happ /usr/local/bin/list_happs /usr/local/bin/uninstall_happ /usr/local/bin/enable_happ /usr/local/bin/disable_happ + +ARG HAPP_CONFIG_FILE_VERSION=0.3.0 + +RUN case "${TARGETARCH}" in \ + amd64) HCF_ARCH="linux-x86_64" ;; \ + arm64) HCF_ARCH="linux-aarch64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + wget "https://github.com/Holo-Host/edgenode/releases/download/tools%2Fhapp_config_file-v${HAPP_CONFIG_FILE_VERSION}/happ_config_file-${HCF_ARCH}" && \ + mv "happ_config_file-${HCF_ARCH}" /usr/local/bin/happ_config_file && \ + chmod +x /usr/local/bin/happ_config_file + +RUN mkdir -p /usr/local/share/holochain +COPY conductor-config-0.6.1.template.yaml /usr/local/share/holochain/conductor-config.template.yaml +COPY logrotate.d/holochain.conf /etc/logrotate.d/holochain.conf +RUN chown -R nonroot:nonroot /usr/local/share/holochain + +COPY s6-overlay-harvester/s6-rc.d/ /etc/s6-overlay/s6-rc.d/ +RUN chmod +x \ + /etc/s6-overlay/s6-rc.d/setup/init.sh \ + /etc/s6-overlay/s6-rc.d/conductor/run \ + /etc/s6-overlay/s6-rc.d/log-harvester/run \ + /etc/s6-overlay/s6-rc.d/logrotate-cron/run + +ENV HC_ADMIN_PORT=4444 +ENV HC_APP_PORT=4445 +ENV HAPP_PATH=/app/unyt.happ +ENV CONFIG_PATH=/etc/log-harvester/config.json + +SHELL ["/bin/sh", "-c"] +EXPOSE 4444 4445 + +ENTRYPOINT ["/init"] diff --git a/docker/LOG_HARVESTER_QUICKSTART.md b/docker/LOG_HARVESTER_QUICKSTART.md new file mode 100644 index 0000000..595a213 --- /dev/null +++ b/docker/LOG_HARVESTER_QUICKSTART.md @@ -0,0 +1,159 @@ +# Log-Harvester Quickstart (Unyt Integration) + +Connect an Edge Node to the Unyt billing pipeline using the harvester container variant. The harvester reads from a `log-collector` service, aggregates usage data, and parks invoices on Unyt Agreements via an embedded Holochain conductor. + +See [Docker README.md](./README.md) for general Edge Node setup. The harvester image is built from [Dockerfile.harvester](./Dockerfile.harvester). + +For upstream docs, see [unytco/log-harvester](https://github.com/unytco/log-harvester). + +## How it works + +``` +log-collector (Cloudflare Worker) + → edgenode-harvester (this container) + ├── Holochain conductor (admin: 4444, app: 4445) + ├── unyt.happ (installed at first startup) + └── log-harvester (Node.js, runs on a daily loop) + → Unyt Agreements (parked spend invoices) +``` + +On first startup the container: +1. Starts the Holochain conductor +2. Installs and enables `unyt.happ` +3. Attaches an app websocket on port 4445 +4. Initializes the harvester config (generates credentials, registers with log-collector) +5. Runs the harvester loop + +On subsequent restarts, credentials are automatically refreshed and `lastInvoice` is preserved so no billing data is lost. + +## Prerequisites + +- Docker installed +- A running `log-collector` instance and its admin secret +- Network access from the container to `COLLECTOR_URL` + +## Step-by-step + +### 1. Pull the image + +```bash +docker login ghcr.io +docker pull ghcr.io/holo-host/edgenode-harvester +``` + +Images are available from [GitHub Packages](https://github.com/Holo-Host/edgenode/pkgs/container/edgenode-harvester). + +The `unyt.happ` is baked in from the latest [unytco/unyt-sandbox](https://github.com/unytco/unyt-sandbox) release. To pin a specific version, build locally: + +```bash +docker buildx build docker/ --file docker/Dockerfile.harvester \ + --build-arg UNYT_HAPP_VERSION=v0.62.0 \ + --secret id=github_token,env=GITHUB_TOKEN \ + --tag my-edgenode-harvester \ + --load +``` + +### 2. Start the container + +```bash +docker run --name harvester -dit \ + -v $(pwd)/holo-data:/data \ + -p 4444:4444 \ + -p 4445:4445 \ + -e COLLECTOR_URL=https://your-log-collector.unyt.dev \ + -e ADMIN_SECRET=your-admin-secret \ + -e LAIR_PASSWORD=your-lair-password \ + ghcr.io/holo-host/edgenode-harvester +``` + +### 3. Watch startup + +First startup takes longer — the conductor needs to initialize its keystore and install the hApp: + +```bash +docker logs -f harvester +# Watch startup.log for progress +docker exec -it harvester tail -f /data/logs/startup.log +``` + +Look for `Starting log-harvester service...` to confirm the harvester is running. + +### 4. Verify + +Check the conductor is running: + +```bash +docker exec -it harvester pgrep holochain +``` + +Check the unyt hApp is installed: + +```bash +docker exec -it harvester hc sandbox call --running 4444 list-apps +``` + +Check the harvester config was initialized: + +```bash +docker exec -it harvester cat /etc/log-harvester/config.json +``` + +Check the harvester is running and reporting: + +```bash +docker exec -it harvester tail -f /data/logs/log-harvester.log +``` + +## Environment variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `COLLECTOR_URL` | Log-collector endpoint URL | (required) | +| `ADMIN_SECRET` | Log-collector admin secret | (required) | +| `LAIR_PASSWORD` | Lair keystore password | (required) | +| `HC_APP_ID` | Installed Unyt hApp ID | `unyt` | +| `HC_NETWORK_SEED` | Network seed for hApp installation | `network-seed` | +| `HC_ADMIN_PORT` | Holochain admin websocket port | `4444` | +| `HC_APP_PORT` | Holochain app websocket port | `4445` | +| `LOG_FOR_TODAY` | Invoice today instead of yesterday (testing only) | `false` | +| `RUST_LOG` | Holochain log level | `info` | + +## Persistent data + +All state is stored under the `/data` volume: + +| Path | Contents | +|------|----------| +| `/data/holochain/` | Conductor state, DHT, keystore | +| `/data/log-harvester/` | Harvester config (`config.json`) | +| `/data/logs/holochain.log` | Conductor logs | +| `/data/logs/log-harvester.log` | Harvester output | +| `/data/logs/startup.log` | Startup sequence log | + +Map `/data` to a named volume or host path to persist across container restarts: + +```bash +-v $(pwd)/holo-data:/data +``` + +## Port reference + +| Port | Purpose | +|------|---------| +| `4444` | Holochain admin websocket | +| `4445` | Holochain app websocket (used by log-harvester) | + +## Troubleshooting + +**Harvester fails to initialize config** +- Check `COLLECTOR_URL` is reachable from the container +- Verify `ADMIN_SECRET` is correct +- Check `/data/logs/startup.log` for the specific error + +**`unyt.happ` install fails** +- Check `/data/logs/startup.log` for hApp installation errors +- The conductor must be fully started before the hApp is installed; the run script waits automatically but a slow keystore initialization can cause a timeout + +**Credentials stale after restart** +- Normal behavior — credentials are automatically refreshed on every restart +- The `lastInvoice` timestamp is preserved so no billing period is skipped diff --git a/docker/README.md b/docker/README.md index 8820b82..e3d3865 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,10 +1,17 @@ # Edge Node Container -A docker container for running Holochain and installing hApps as always-on nodes. For Unyt log-sender integration, see [LOG_SENDER_QUICKSTART.md](./LOG_SENDER_QUICKSTART.md). +Docker containers for running Holochain and installing hApps as always-on nodes. + +Two variants are available: + +| Image | Description | +|-------|-------------| +| `ghcr.io/holo-host/edgenode` | Standard edge node with `log-sender` for Unyt resource accounting. See [LOG_SENDER_QUICKSTART.md](./LOG_SENDER_QUICKSTART.md). | +| `ghcr.io/holo-host/edgenode-harvester` | Harvester variant with `log-harvester` for Unyt invoice aggregation. See [LOG_HARVESTER_QUICKSTART.md](./LOG_HARVESTER_QUICKSTART.md). | ## Container Commands -These are available inside the container: +These commands are available inside both container variants: - `install_happ [-p ] [node_name]` -- Install a hApp from a config file - `uninstall_happ [-p ] ` -- Uninstall a hApp @@ -12,6 +19,8 @@ These are available inside the container: - `disable_happ [-p ] ` -- Disable an installed hApp - `list_happs [-p ]` -- List installed hApps - `happ_config_file` -- Create/validate hApp config files (run with `--help` for usage) + +Standard variant (`edgenode`) only: - `log_tool ` -- Manage the Unyt log-sender service (see [LOG_SENDER_QUICKSTART.md](./LOG_SENDER_QUICKSTART.md)) - `wdocker ` -- Manage always-on Moss group nodes (see [EdgeNode Moss Guide](https://holo.host/files/EdgeNodeMossGuide.pdf)) - `wdaemon` -- wdocker background daemon @@ -105,10 +114,19 @@ Paths are symlinked into the `/data` volume for persistence: ## Process Management - **s6-overlay** runs as PID 1, supervising all services with automatic restart on crash -- Services: `conductor`, `log-sender`, `logrotate-cron` (all longruns), plus `setup` oneshot on startup - All processes run as nonroot (UID 65532) - Log rotation via logrotate (daily, 7-day retention, gzip compression) +Services by variant: + +| Service | `edgenode` | `edgenode-harvester` | +|---------|-----------|---------------------| +| `setup` (oneshot) | ✓ | ✓ | +| `conductor` (longrun) | ✓ | ✓ | +| `logrotate-cron` (longrun) | ✓ | ✓ | +| `log-sender` (longrun) | ✓ | — | +| `log-harvester` (longrun) | — | ✓ | + ## Development ### Sandbox mode diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2a6d1ff..36a0250 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -42,9 +42,47 @@ services: timeout: 10s retries: 3 + edgenode-harvester: + image: ${HARVESTER_IMAGE:-local-edgenode-harvester} + build: + context: . + dockerfile: Dockerfile.harvester + secrets: + - github_token + depends_on: + log-collector: + condition: service_healthy + environment: + - COLLECTOR_URL=${COLLECTOR_URL:-http://log-collector:8787} + - ADMIN_SECRET=${ADMIN_SECRET:-test_admin_secret} + - LAIR_PASSWORD=${LAIR_PASSWORD:-password} + - HC_APP_ID=${HC_APP_ID:-unyt} + - HC_NETWORK_SEED=${HC_NETWORK_SEED:-network-seed} + - LOG_FOR_TODAY=${LOG_FOR_TODAY:-false} + - RUST_LOG=${RUST_LOG:-info} + volumes: + - holo-data-harvester:/data + - holo-config-harvester:/etc/log-harvester + networks: + - test-net + healthcheck: + test: ["CMD", "pgrep", "holochain"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + networks: test-net: +secrets: + github_token: + environment: GITHUB_TOKEN + volumes: holo-data-test: driver: local + holo-data-harvester: + driver: local + holo-config-harvester: + driver: local diff --git a/docker/s6-overlay-harvester/s6-rc.d/conductor/dependencies.d/setup b/docker/s6-overlay-harvester/s6-rc.d/conductor/dependencies.d/setup new file mode 100644 index 0000000..e69de29 diff --git a/docker/s6-overlay-harvester/s6-rc.d/conductor/run b/docker/s6-overlay-harvester/s6-rc.d/conductor/run new file mode 100644 index 0000000..10a9385 --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/conductor/run @@ -0,0 +1,6 @@ +#!/bin/sh +if [ "${CONDUCTOR_MODE:-}" = "false" ]; then + exec tail -f /dev/null +fi +echo "Starting holochain conductor..." >> /data/logs/startup.log +gosu nonroot sh -c 'yes | holochain --piped --config-path /etc/holochain/conductor-config.yaml' 2>&1 | tee -a /data/logs/holochain.log diff --git a/docker/s6-overlay-harvester/s6-rc.d/conductor/type b/docker/s6-overlay-harvester/s6-rc.d/conductor/type new file mode 100644 index 0000000..5883cff --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/conductor/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay-harvester/s6-rc.d/log-harvester/dependencies.d/setup b/docker/s6-overlay-harvester/s6-rc.d/log-harvester/dependencies.d/setup new file mode 100644 index 0000000..e69de29 diff --git a/docker/s6-overlay-harvester/s6-rc.d/log-harvester/run b/docker/s6-overlay-harvester/s6-rc.d/log-harvester/run new file mode 100644 index 0000000..f8924f1 --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/log-harvester/run @@ -0,0 +1,76 @@ +#!/bin/sh +set -e + +APP_ID="${HC_APP_ID:-unyt}" +NETWORK_SEED="${HC_NETWORK_SEED:-network-seed}" + +# Wait for holochain conductor to be ready +echo "Waiting for holochain conductor..." >> /data/logs/startup.log +while ! hc sandbox call --running "${HC_ADMIN_PORT}" list-apps > /dev/null 2>&1; do + sleep 5 +done +echo "Holochain conductor is ready." >> /data/logs/startup.log + +# Install unyt.happ if not already installed +if [ -f "${HAPP_PATH}" ]; then + if ! hc sandbox call --running "${HC_ADMIN_PORT}" list-apps 2>/dev/null | grep -q "${APP_ID}"; then + echo "Installing ${APP_ID} happ..." >> /data/logs/startup.log + hc sandbox call --running "${HC_ADMIN_PORT}" install-app "${HAPP_PATH}" \ + --app-id "${APP_ID}" "${NETWORK_SEED}" >> /data/logs/startup.log 2>&1 || true + hc sandbox call --running "${HC_ADMIN_PORT}" enable-app "${APP_ID}" >> /data/logs/startup.log 2>&1 || true + fi +else + echo "WARNING: ${HAPP_PATH} not found — skipping happ install" >> /data/logs/startup.log +fi + +# Attach app websocket on HC_APP_PORT (idempotent) +hc sandbox call --running "${HC_ADMIN_PORT}" add-app-ws "${HC_APP_PORT}" \ + --allowed-origins "log-harvester" >> /data/logs/startup.log 2>&1 || true + +# Wait a moment for keystore to be fully ready before touching credentials +sleep 3 + +# Initialize config on first run; refresh credentials on restart +if [ ! -f "${CONFIG_PATH}" ]; then + echo "Initializing harvester config..." >> /data/logs/startup.log + node /app/dist/log-harvester.js init \ + --config "${CONFIG_PATH}" \ + --url "${COLLECTOR_URL}" \ + --admin "${ADMIN_SECRET}" \ + --hc-admin "${HC_ADMIN_PORT}" \ + --hc-app "${HC_APP_PORT}" \ + --app-id "${APP_ID}" >> /data/logs/startup.log 2>&1 +else + echo "Refreshing harvester credentials..." >> /data/logs/startup.log + cp "${CONFIG_PATH}" "${CONFIG_PATH}.backup" + LAST_INVOICE="" + if command -v jq > /dev/null 2>&1; then + LAST_INVOICE=$(jq -r '.lastInvoice // empty' "${CONFIG_PATH}" 2>/dev/null || true) + fi + rm "${CONFIG_PATH}" + node /app/dist/log-harvester.js init \ + --config "${CONFIG_PATH}" \ + --url "${COLLECTOR_URL}" \ + --admin "${ADMIN_SECRET}" \ + --hc-admin "${HC_ADMIN_PORT}" \ + --hc-app "${HC_APP_PORT}" \ + --app-id "${APP_ID}" >> /data/logs/startup.log 2>&1 && \ + rm -f "${CONFIG_PATH}.backup" || \ + mv "${CONFIG_PATH}.backup" "${CONFIG_PATH}" + if [ -n "${LAST_INVOICE}" ] && command -v jq > /dev/null 2>&1; then + jq ".lastInvoice = ${LAST_INVOICE}" "${CONFIG_PATH}" > "${CONFIG_PATH}.tmp" && \ + mv "${CONFIG_PATH}.tmp" "${CONFIG_PATH}" || true + fi +fi + +echo "Starting log-harvester service..." >> /data/logs/startup.log + +TODAY_FLAG="" +if [ "${LOG_FOR_TODAY:-false}" = "true" ]; then + TODAY_FLAG="--today" +fi + +exec gosu nonroot node /app/dist/log-harvester.js exec \ + --config "${CONFIG_PATH}" \ + --loop \ + ${TODAY_FLAG} >> /data/logs/log-harvester.log 2>&1 diff --git a/docker/s6-overlay-harvester/s6-rc.d/log-harvester/type b/docker/s6-overlay-harvester/s6-rc.d/log-harvester/type new file mode 100644 index 0000000..5883cff --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/log-harvester/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/dependencies.d/setup b/docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/dependencies.d/setup new file mode 100644 index 0000000..e69de29 diff --git a/docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/run b/docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/run new file mode 100644 index 0000000..4ff53e6 --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/run @@ -0,0 +1,5 @@ +#!/bin/sh +while true; do + logrotate /etc/logrotate.d/holochain.conf + sleep 86400 +done diff --git a/docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/type b/docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/type new file mode 100644 index 0000000..5883cff --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/logrotate-cron/type @@ -0,0 +1 @@ +longrun diff --git a/docker/s6-overlay-harvester/s6-rc.d/setup/init.sh b/docker/s6-overlay-harvester/s6-rc.d/setup/init.sh new file mode 100644 index 0000000..c12f2e8 --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/setup/init.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -eu + +# Create persistent storage directories +mkdir -p /data/holochain/etc /data/holochain/var /data/logs /data/log-harvester +touch /data/logs/startup.log + +# Fix ownership for copied files in nonroot home +chown -R nonroot:nonroot /home/nonroot +echo "Chowned /home/nonroot contents" >> /data/logs/startup.log 2>/dev/null || true + +# Ensure parent directories exist for symlinks +mkdir -p /var/local/lib + +# Create symlinks for persistent storage +ln -sfn /data/holochain/etc /etc/holochain +ln -sfn /data/log-harvester /etc/log-harvester +ln -sfn /data/holochain/var /var/local/lib/holochain +mkdir -p /data/holochain/var/ks /data/holochain/tmp /data/holochain/var/tmp +chown -R nonroot:nonroot /data +chown -R nonroot:nonroot /data/holochain +chmod 700 /data/holochain/var/ks +chmod 755 /data/holochain/tmp /data/holochain/var/tmp + +# Copy conductor config template +cp /usr/local/share/holochain/conductor-config.template.yaml /etc/holochain/conductor-config.yaml + +# Validate admin port configuration +if ! grep -q "port: 4444" /etc/holochain/conductor-config.yaml; then + echo "ERROR: Conductor config must use admin port 4444" >&2 + exit 1 +fi + +# Validate keystore configuration for lair_server_in_proc +if ! grep -q "keystore:" /etc/holochain/conductor-config.yaml || ! grep -q "type: lair_server_in_proc" /etc/holochain/conductor-config.yaml; then + echo "ERROR: Conductor config must have keystore with type: lair_server_in_proc" >&2 + exit 1 +fi + +echo "Setup complete." >> /data/logs/startup.log diff --git a/docker/s6-overlay-harvester/s6-rc.d/user/contents.d/conductor b/docker/s6-overlay-harvester/s6-rc.d/user/contents.d/conductor new file mode 100644 index 0000000..e69de29 diff --git a/docker/s6-overlay-harvester/s6-rc.d/user/contents.d/log-harvester b/docker/s6-overlay-harvester/s6-rc.d/user/contents.d/log-harvester new file mode 100644 index 0000000..e69de29 diff --git a/docker/s6-overlay-harvester/s6-rc.d/user/contents.d/logrotate-cron b/docker/s6-overlay-harvester/s6-rc.d/user/contents.d/logrotate-cron new file mode 100644 index 0000000..e69de29 diff --git a/docker/s6-overlay-harvester/s6-rc.d/user/type b/docker/s6-overlay-harvester/s6-rc.d/user/type new file mode 100644 index 0000000..757b422 --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/user/type @@ -0,0 +1 @@ +bundle From de039361d191256f1c486bf590f634f5b610bb7c Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 14:11:15 +0000 Subject: [PATCH 02/21] test: add Tier 1 integration tests for edgenode-harvester Adds BATS test files and a test runner for the harvester container variant: - harvester_startup.bats: verifies conductor start, unyt.happ install, app-ws on 4445, config initialization, and log-harvester service start - harvester_process.bats: verifies holochain and node run as nonroot - run_harvester_tests.sh: runner analogous to run_tests_multi.sh, builds the harvester image with GITHUB_TOKEN secret and waits for full startup - pr-checks.yml: adds test-harvester-image CI job running on docker/** PRs Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 28 +++++++++ docker/run_harvester_tests.sh | 96 +++++++++++++++++++++++++++++ docker/tests/harvester_process.bats | 18 ++++++ docker/tests/harvester_startup.bats | 44 +++++++++++++ 4 files changed, 186 insertions(+) create mode 100755 docker/run_harvester_tests.sh create mode 100644 docker/tests/harvester_process.bats create mode 100644 docker/tests/harvester_startup.bats diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index dc70701..dbb95ec 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -33,3 +33,31 @@ jobs: - name: Run tests run: | CI_RELEASE_TEST=true ./docker/run_tests_multi.sh local-edgenode + + test-harvester-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build harvester image for testing + env: + GITHUB_TOKEN: ${{ secrets.HARVESTER_REPO_TOKEN }} + run: | + docker buildx build docker/ --file docker/Dockerfile.harvester \ + --platform linux/amd64 \ + --secret id=github_token,env=GITHUB_TOKEN \ + --load \ + --tag local-edgenode-harvester + + - name: Run harvester tests + env: + GITHUB_TOKEN: ${{ secrets.HARVESTER_REPO_TOKEN }} + run: | + CI_RELEASE_TEST=true ./docker/run_harvester_tests.sh local-edgenode-harvester diff --git a/docker/run_harvester_tests.sh b/docker/run_harvester_tests.sh new file mode 100755 index 0000000..22afa55 --- /dev/null +++ b/docker/run_harvester_tests.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +set -ex + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +IMAGE_NAME="${1:-local-edgenode-harvester}" +SERVICE_NAME="edgenode-harvester" +CLEANUP="${CLEANUP:-true}" + +echo "Testing harvester image: $IMAGE_NAME" + +cleanup() { + if [[ "$CLEANUP" == "true" ]]; then + echo "Cleaning up..." + docker compose down -v --remove-orphans + fi +} +trap cleanup EXIT + +# Build local image unless running in CI release test mode +if [[ "$CI_RELEASE_TEST" != "true" ]] && [[ "$IMAGE_NAME" == local-edgenode-harvester* ]]; then + echo "Building local harvester image: $IMAGE_NAME" + docker build \ + --secret id=github_token,env=GITHUB_TOKEN \ + -t "$IMAGE_NAME" \ + -f Dockerfile.harvester \ + . +fi + +export HARVESTER_IMAGE="$IMAGE_NAME" +export IMAGE_NAME +export SERVICE_NAME +export SCRIPT_DIR +export COMPOSE_PROJECT_NAME="edgenode" + +# Start services +echo "Starting services..." +docker compose up --build -d log-collector "$SERVICE_NAME" + +# Wait for log-collector +echo "Waiting for log-collector to be healthy..." +MAX_WAIT=60 +WAIT_TIME=0 +while [ $WAIT_TIME -lt $MAX_WAIT ]; do + if docker compose ps log-collector | grep -q "healthy"; then + echo "Log-collector is healthy" + break + fi + echo "Waiting for log-collector... ($WAIT_TIME/$MAX_WAIT seconds)" + sleep 5 + WAIT_TIME=$((WAIT_TIME + 5)) +done + +# Harvester has a longer startup — conductor init + happ install +echo "Waiting for harvester to start (up to 120s)..." +MAX_WAIT=120 +WAIT_TIME=0 +while [ $WAIT_TIME -lt $MAX_WAIT ]; do + if docker compose exec -T "$SERVICE_NAME" test -f /data/logs/startup.log 2>/dev/null && \ + docker compose exec -T "$SERVICE_NAME" grep -q "Starting log-harvester service" /data/logs/startup.log 2>/dev/null; then + echo "Harvester is ready" + break + fi + echo "Waiting for harvester... ($WAIT_TIME/$MAX_WAIT seconds)" + sleep 10 + WAIT_TIME=$((WAIT_TIME + 10)) +done + +# Resolve actual container name +ACTUAL_CONTAINER=$(docker compose ps -q "$SERVICE_NAME" 2>/dev/null | head -n 1) +if [ -n "$ACTUAL_CONTAINER" ]; then + export CONTAINER_NAME="$ACTUAL_CONTAINER" + echo "Found container: $CONTAINER_NAME for service: $SERVICE_NAME" +else + export CONTAINER_NAME="edgenode-${SERVICE_NAME}-1" + echo "Warning: container not found, using fallback: $CONTAINER_NAME" +fi + +# Run harvester-specific tests only +echo "Running harvester tests..." +set +e +./tests/libs/bats/bin/bats tests/harvester_startup.bats tests/harvester_process.bats +TEST_EXIT_CODE=$? +set -e + +if [ $TEST_EXIT_CODE -ne 0 ]; then + echo "Tests failed. Printing container logs..." + docker compose logs "$SERVICE_NAME" + echo "--- startup.log ---" + docker compose exec -T "$SERVICE_NAME" cat /data/logs/startup.log 2>/dev/null || true +fi + +echo "Harvester test execution completed with exit code: $TEST_EXIT_CODE" +exit $TEST_EXIT_CODE diff --git a/docker/tests/harvester_process.bats b/docker/tests/harvester_process.bats new file mode 100644 index 0000000..d1e13cd --- /dev/null +++ b/docker/tests/harvester_process.bats @@ -0,0 +1,18 @@ +#!/usr/bin/env bats + +load 'libs/bats-support/load' +load 'libs/bats-assert/load' + +HARVESTER_SERVICE="${SERVICE_NAME:-edgenode-harvester}" + +@test "Holochain process runs as nonroot" { + run docker compose exec -T "$HARVESTER_SERVICE" \ + sh -c "ps aux | grep -E 'nonroot.*holochain'" + assert_success +} + +@test "log-harvester node process runs as nonroot" { + run docker compose exec -T "$HARVESTER_SERVICE" \ + sh -c "ps aux | grep -E 'nonroot.*node'" + assert_success +} diff --git a/docker/tests/harvester_startup.bats b/docker/tests/harvester_startup.bats new file mode 100644 index 0000000..069a1cd --- /dev/null +++ b/docker/tests/harvester_startup.bats @@ -0,0 +1,44 @@ +#!/usr/bin/env bats + +load 'libs/bats-support/load' +load 'libs/bats-assert/load' + +HARVESTER_SERVICE="${SERVICE_NAME:-edgenode-harvester}" + +@test "Conductor starts successfully" { + run docker compose logs "$HARVESTER_SERVICE" + assert_output --partial "Conductor ready." +} + +@test "unyt.happ is installed" { + run docker compose exec -T "$HARVESTER_SERVICE" \ + hc sandbox call --running 4444 list-apps + assert_success + assert_output --partial "unyt" +} + +@test "App websocket attached on port 4445" { + run docker compose exec -T "$HARVESTER_SERVICE" \ + sh -c "cat /data/logs/startup.log" + assert_success + assert_output --partial "add-app-ws" +} + +@test "Harvester config is initialized" { + run docker compose exec -T "$HARVESTER_SERVICE" \ + test -f /etc/log-harvester/config.json + assert_success +} + +@test "Harvester config contains droneId" { + run docker compose exec -T "$HARVESTER_SERVICE" \ + jq -e '.droneId' /etc/log-harvester/config.json + assert_success +} + +@test "log-harvester service starts" { + run docker compose exec -T "$HARVESTER_SERVICE" \ + sh -c "cat /data/logs/startup.log" + assert_success + assert_output --partial "Starting log-harvester service..." +} From a41db8b0e6b02f5d9565233d3120df6690612092 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:20:18 +0000 Subject: [PATCH 03/21] =?UTF-8?q?test:=20add=20e2e=20pipeline=20test=20for?= =?UTF-8?q?=20log-sender=20=E2=86=92=20log-collector=20=E2=86=92=20harvest?= =?UTF-8?q?er?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds harvester_e2e.bats which submits real signed metrics via log-sender, verifies they reach D1, then runs the harvester (--today --dry-run) and asserts "fetched metrics count" > 0 and "Successfully invoiced logs." Trims harvester_integration.bats to two lightweight connectivity checks. Extends run_harvester_tests.sh to start edgenode and wait for its Holochain conductor before running the e2e test. Co-Authored-By: Claude Sonnet 4.6 --- docker/run_harvester_tests.sh | 52 ++++++++--- docker/tests/harvester_e2e.bats | 113 ++++++++++++++++++++++++ docker/tests/harvester_integration.bats | 34 +++++++ 3 files changed, 187 insertions(+), 12 deletions(-) create mode 100644 docker/tests/harvester_e2e.bats create mode 100644 docker/tests/harvester_integration.bats diff --git a/docker/run_harvester_tests.sh b/docker/run_harvester_tests.sh index 22afa55..9c30bac 100755 --- a/docker/run_harvester_tests.sh +++ b/docker/run_harvester_tests.sh @@ -19,14 +19,24 @@ cleanup() { } trap cleanup EXIT -# Build local image unless running in CI release test mode +# Build local image unless running in CI release test mode. +# Set FORCE_REBUILD=true to rebuild even if the image already exists locally. if [[ "$CI_RELEASE_TEST" != "true" ]] && [[ "$IMAGE_NAME" == local-edgenode-harvester* ]]; then - echo "Building local harvester image: $IMAGE_NAME" - docker build \ - --secret id=github_token,env=GITHUB_TOKEN \ - -t "$IMAGE_NAME" \ - -f Dockerfile.harvester \ - . + if [[ "$FORCE_REBUILD" != "true" ]] && docker image inspect "$IMAGE_NAME" >/dev/null 2>&1; then + echo "Using existing local image: $IMAGE_NAME (set FORCE_REBUILD=true to rebuild)" + else + echo "Building local harvester image: $IMAGE_NAME" + SECRET_ARGS="" + if [ -n "$GITHUB_TOKEN" ]; then + SECRET_ARGS="--secret id=github_token,env=GITHUB_TOKEN" + fi + # shellcheck disable=SC2086 + docker build \ + $SECRET_ARGS \ + -t "$IMAGE_NAME" \ + -f Dockerfile.harvester \ + . + fi fi export HARVESTER_IMAGE="$IMAGE_NAME" @@ -35,9 +45,13 @@ export SERVICE_NAME export SCRIPT_DIR export COMPOSE_PROJECT_NAME="edgenode" -# Start services +# Build log-collector separately so Docker layer cache is used on subsequent runs +echo "Building log-collector..." +docker compose build log-collector + +# Start services without rebuilding (log-collector already built above) echo "Starting services..." -docker compose up --build -d log-collector "$SERVICE_NAME" +docker compose up -d log-collector edgenode "$SERVICE_NAME" # Wait for log-collector echo "Waiting for log-collector to be healthy..." @@ -53,10 +67,24 @@ while [ $WAIT_TIME -lt $MAX_WAIT ]; do WAIT_TIME=$((WAIT_TIME + 5)) done -# Harvester has a longer startup — conductor init + happ install -echo "Waiting for harvester to start (up to 120s)..." +# Wait for edgenode Holochain conductor (needed for log-sender e2e test) +echo "Waiting for edgenode to start (up to 120s)..." MAX_WAIT=120 WAIT_TIME=0 +while [ $WAIT_TIME -lt $MAX_WAIT ]; do + if docker compose exec -T edgenode pgrep holochain > /dev/null 2>&1; then + echo "Edgenode is ready" + break + fi + echo "Waiting for edgenode... ($WAIT_TIME/$MAX_WAIT seconds)" + sleep 5 + WAIT_TIME=$((WAIT_TIME + 5)) +done + +# Harvester has a longer startup — conductor keystore init + happ install + config init +echo "Waiting for harvester to start (up to 300s)..." +MAX_WAIT=300 +WAIT_TIME=0 while [ $WAIT_TIME -lt $MAX_WAIT ]; do if docker compose exec -T "$SERVICE_NAME" test -f /data/logs/startup.log 2>/dev/null && \ docker compose exec -T "$SERVICE_NAME" grep -q "Starting log-harvester service" /data/logs/startup.log 2>/dev/null; then @@ -81,7 +109,7 @@ fi # Run harvester-specific tests only echo "Running harvester tests..." set +e -./tests/libs/bats/bin/bats tests/harvester_startup.bats tests/harvester_process.bats +./tests/libs/bats/bin/bats tests/harvester_startup.bats tests/harvester_process.bats tests/harvester_integration.bats tests/harvester_e2e.bats TEST_EXIT_CODE=$? set -e diff --git a/docker/tests/harvester_e2e.bats b/docker/tests/harvester_e2e.bats new file mode 100644 index 0000000..5cf51c1 --- /dev/null +++ b/docker/tests/harvester_e2e.bats @@ -0,0 +1,113 @@ +#!/usr/bin/env bats + +load 'libs/bats-support/load' +load 'libs/bats-assert/load' + +# End-to-end pipeline test: edgenode (log-sender) -> log-collector -> edgenode-harvester +# +# Requires both edgenode and edgenode-harvester services to be running. +# Run via run_harvester_tests.sh which starts both services. + +EDGENODE_SERVICE="edgenode" +HARVESTER_SERVICE="${SERVICE_NAME:-edgenode-harvester}" +ADMIN_SECRET="${ADMIN_SECRET:-test_admin_secret}" +# Match the docker-compose default so install_happ's DNA registration (which uses +# the startup log-sender config) aligns with the UNYT key on our test metrics. +UNYT_PUB_KEY="${LOG_SENDER_UNYT_PUB_KEY:-uhCAkDM-p0oBsRJn5Ebpk8c_TNkrp2NEwF9C5ppJq8cE77I-n3qfO}" +LOG_COLLECTOR_URL="http://log-collector:8787" +CONFIG_PATH="/data/log-harvester/config.json" + +setup() { + if ! docker compose ps "$EDGENODE_SERVICE" 2>/dev/null | grep -q "running\|Up"; then + skip "edgenode service is not running" + fi + if ! docker compose ps "$HARVESTER_SERVICE" 2>/dev/null | grep -q "running\|Up"; then + skip "edgenode-harvester service is not running" + fi +} + +@test "full pipeline: log-sender submits metrics that harvester fetches and invoices" { + local E2E_CONFIG="/etc/log-sender/e2e_harvester_test.json" + local E2E_LOG_DIR="/data/logs/e2e_harvester_test" + + # --- Setup: clean slate --- + + docker compose exec -T -u nonroot "$EDGENODE_SERVICE" rm -f "$E2E_CONFIG" 2>/dev/null || true + docker compose exec -T "$HARVESTER_SERVICE" rm -f /data/e2e-harvest-config.json 2>/dev/null || true + + # Wipe all metrics and invoice periods so the harvester only processes the small + # set of metrics we submit in this test — avoids paginating through hundreds of + # metrics from the continuously running edgenode log-sender. + docker compose exec -T log-collector \ + npx --yes wrangler d1 execute log-collector-db \ + --command="DELETE FROM metrics; DELETE FROM invoice_periods; DELETE FROM dna_registrations;" 2>/dev/null || true + + # --- Step 1: Submit real signed metrics via log-sender --- + + run docker compose exec -T -u nonroot "$EDGENODE_SERVICE" mkdir -p "$E2E_LOG_DIR" + assert_success + + local current_time=$(( $(date +%s) * 1000000 )) + run docker compose exec -T -u nonroot "$EDGENODE_SERVICE" \ + sh -c "echo '{\"k\":\"fetchedOps\",\"t\":\"${current_time}\",\"count\":5,\"latency\":50}' > ${E2E_LOG_DIR}/metrics.jsonl" + assert_success + + run docker compose exec -T -u nonroot "$EDGENODE_SERVICE" log-sender init \ + --config-file "$E2E_CONFIG" \ + --endpoint "$LOG_COLLECTOR_URL" \ + --unyt-pub-key "$UNYT_PUB_KEY" \ + --report-path "$E2E_LOG_DIR/" \ + --conductor-config-path /etc/holochain/conductor-config.yaml \ + --report-interval-seconds 2 + assert_success + + # install_happ registers the relay DNA in the log-collector under the startup + # log-sender config's drone (which uses UNYT_PUB_KEY = uhCAkDM-...). + # This populates dna_registrations so get-registered-dna succeeds for that UNYT key. + local relay_json="${SCRIPT_DIR:-${BATS_TEST_DIRNAME}/..}/relay.json" + docker compose cp "$relay_json" "${EDGENODE_SERVICE}:/home/nonroot/" + run docker compose exec -T -u nonroot "$EDGENODE_SERVICE" \ + sh -c 'cd /home/nonroot && install_happ relay.json test-node' + assert_success + + run docker compose exec -T -u nonroot "$EDGENODE_SERVICE" \ + timeout 25 log-sender service --config-file "$E2E_CONFIG" + # timeout exit is expected; what matters is that metrics were submitted + + # --- Step 2: Verify fresh metrics reached D1 --- + + local count + count=$(docker compose exec -T log-collector \ + npx --yes wrangler d1 execute log-collector-db \ + --command="SELECT COUNT(*) as total FROM metrics;" 2>/dev/null \ + | grep -o '"total": [0-9]*' | grep -o '[0-9]*' | head -1 || echo "0") + + [[ "$count" -gt 0 ]] || fail "No new metrics found in D1 after log-sender run (count=$count)" + + # --- Step 3: Run harvester one-shot against the fresh data --- + + # Copy config with lastInvoice=0 to bypass the 24h gate and avoid the lockfile + # held by the running service. + run docker compose exec -T "$HARVESTER_SERVICE" \ + sh -c "jq '.lastInvoice = 0' ${CONFIG_PATH} > /data/e2e-harvest-config.json && chown nonroot:nonroot /data/e2e-harvest-config.json" + assert_success + + # --today : query today's UTC range (covers our freshly submitted metrics) + # --dry-run : skip the Holochain create_parked_spend call; still calls + # get-registered-dna and prints "Successfully invoiced logs." + run docker compose exec -T "$HARVESTER_SERVICE" \ + gosu nonroot node /app/dist/log-harvester.js exec \ + --config /data/e2e-harvest-config.json \ + --today \ + --dry-run + assert_success + + # --- Step 4: Assert the full pipeline completed end-to-end --- + + # Harvester fetched at least one metric from log-collector + assert_output --partial '"fetched metrics count"' + echo "$output" | grep '"fetched metrics count"' | grep -qv ',0\]' + + # Harvester resolved the full invoice cycle (dry-run path skips Holochain call) + assert_output --partial '"Successfully invoiced logs."' +} diff --git a/docker/tests/harvester_integration.bats b/docker/tests/harvester_integration.bats new file mode 100644 index 0000000..aa46ad1 --- /dev/null +++ b/docker/tests/harvester_integration.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +load 'libs/bats-support/load' +load 'libs/bats-assert/load' + +# Lightweight connectivity checks: harvester container <-> log-collector. +# For the full end-to-end pipeline test see harvester_e2e.bats. + +HARVESTER_SERVICE="${SERVICE_NAME:-edgenode-harvester}" +ADMIN_SECRET="${ADMIN_SECRET:-test_admin_secret}" + +setup() { + if ! docker compose ps "$HARVESTER_SERVICE" 2>/dev/null | grep -q "running\|Up"; then + skip "edgenode-harvester service is not running" + fi +} + +@test "harvester network can reach log-collector" { + run docker compose exec -T "$HARVESTER_SERVICE" \ + curl -sf http://log-collector:8787/ + assert_success +} + +@test "log-collector /logs endpoint is accessible with harvester admin credentials" { + local NOW_MS START_MS END_MS + NOW_MS=$(date +%s%3N) + START_MS=$(( NOW_MS - 3600000 )) + END_MS=$(( NOW_MS + 3600000 )) + + run docker compose exec -T "$HARVESTER_SERVICE" \ + sh -c "curl -sf -H 'X-Admin-Secret: ${ADMIN_SECRET}' 'http://log-collector:8787/logs?startTime=${START_MS}&endTime=${END_MS}&limit=1'" + assert_success + assert_output --partial '"success":true' +} From b5ad7e16a976527706fe0967e82cdb392e23c3d2 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:23:23 +0000 Subject: [PATCH 04/21] docs: document harvester test suite in TESTING.md and quickstart Co-Authored-By: Claude Sonnet 4.6 --- docker/LOG_HARVESTER_QUICKSTART.md | 12 ++++++++++-- docker/TESTING.md | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docker/LOG_HARVESTER_QUICKSTART.md b/docker/LOG_HARVESTER_QUICKSTART.md index 595a213..872cbf1 100644 --- a/docker/LOG_HARVESTER_QUICKSTART.md +++ b/docker/LOG_HARVESTER_QUICKSTART.md @@ -95,7 +95,7 @@ docker exec -it harvester hc sandbox call --running 4444 list-apps Check the harvester config was initialized: ```bash -docker exec -it harvester cat /etc/log-harvester/config.json +docker exec -it harvester cat /data/log-harvester/config.json ``` Check the harvester is running and reporting: @@ -115,7 +115,7 @@ docker exec -it harvester tail -f /data/logs/log-harvester.log | `HC_NETWORK_SEED` | Network seed for hApp installation | `network-seed` | | `HC_ADMIN_PORT` | Holochain admin websocket port | `4444` | | `HC_APP_PORT` | Holochain app websocket port | `4445` | -| `LOG_FOR_TODAY` | Invoice today instead of yesterday (testing only) | `false` | +| `LOG_FOR_TODAY` | Query today's UTC range instead of yesterday's (useful when testing same-day submissions) | `false` | | `RUST_LOG` | Holochain log level | `info` | ## Persistent data @@ -143,6 +143,14 @@ Map `/data` to a named volume or host path to persist across container restarts: | `4444` | Holochain admin websocket | | `4445` | Holochain app websocket (used by log-harvester) | +## Running the test suite + +```bash +./run_harvester_tests.sh +``` + +Builds the image, starts all required services, and runs startup, process, connectivity, and end-to-end pipeline tests. See [TESTING.md](./TESTING.md) for details. + ## Troubleshooting **Harvester fails to initialize config** diff --git a/docker/TESTING.md b/docker/TESTING.md index 32a90a2..01b7246 100644 --- a/docker/TESTING.md +++ b/docker/TESTING.md @@ -42,3 +42,24 @@ Create a new `.bats` file in the `tests/` directory. It will be picked up automa ## CI Integration `run_tests_multi.sh` exits with a non-zero status if any test fails. Tests run against the amd64 build before the multi-platform push in the release workflow. + +## Harvester Tests + +The `edgenode-harvester` variant has its own test runner: + +```bash +./run_harvester_tests.sh +``` + +This builds the harvester image, starts `log-collector`, `edgenode`, and `edgenode-harvester`, waits for all three to be ready, then runs: + +- `harvester_startup.bats`: Conductor ready, hApp installed, config initialized, service started. +- `harvester_process.bats`: Holochain and Node.js processes run as `nonroot`. +- `harvester_integration.bats`: Connectivity checks — harvester can reach log-collector and query `/logs`. +- `harvester_e2e.bats`: Full pipeline — log-sender submits signed metrics to log-collector, harvester fetches and invoices them (`--today --dry-run`), asserts `"Successfully invoiced logs."`. + +To test against a pre-built image instead of building locally: + +```bash +CI_RELEASE_TEST=true ./run_harvester_tests.sh ghcr.io/holo-host/edgenode-harvester:v1.2.3 +``` From aebcf552f8e46f0a6c5ba5686f6485aab36d59c7 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:26:34 +0000 Subject: [PATCH 05/21] fix: move config to /data, fix s6 shebangs, add GHA build caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CONFIG_PATH: /etc/log-harvester → /data/log-harvester (volume-mounted, survives restarts; drops now-redundant holo-config-harvester volume) - s6 run scripts: shebang → #!/command/with-contenv so HC_* env vars are available inside the service - log-harvester/run: remove add-app-ws call (handled by harvester init); add chown after both init and refresh paths - Dockerfile.harvester: GITHUB_TOKEN secret now optional — falls back to unauthenticated clone when git credentials already grant access - pr-checks.yml: switch to docker/build-push-action with GHA layer cache for edgenode, edgenode-harvester, and log-collector builds - run_tests_multi.sh: build log-collector separately before compose up to preserve layer cache across repeat runs - harvester_startup.bats: update config path and websocket test to match Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 48 ++++++++++++++----- docker/Dockerfile.harvester | 14 ++++-- docker/docker-compose.yml | 3 -- docker/run_tests_multi.sh | 8 +++- .../s6-rc.d/conductor/run | 2 +- .../s6-rc.d/log-harvester/run | 8 ++-- .../s6-overlay-harvester/s6-rc.d/setup/type | 1 + docker/s6-overlay-harvester/s6-rc.d/setup/up | 2 + docker/tests/harvester_startup.bats | 11 ++--- 9 files changed, 63 insertions(+), 34 deletions(-) create mode 100644 docker/s6-overlay-harvester/s6-rc.d/setup/type create mode 100755 docker/s6-overlay-harvester/s6-rc.d/setup/up diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index dbb95ec..093293b 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -23,12 +23,24 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Build log-collector (cached) + uses: docker/build-push-action@v5 + with: + context: docker/log-collector + load: true + tags: edgenode-log-collector:latest + cache-from: type=gha,scope=log-collector + cache-to: type=gha,mode=max,scope=log-collector + - name: Build image for testing - run: | - docker buildx build docker/ --file docker/Dockerfile \ - --platform linux/amd64 \ - --load \ - --tag local-edgenode + uses: docker/build-push-action@v5 + with: + context: docker/ + file: docker/Dockerfile + load: true + tags: local-edgenode + cache-from: type=gha,scope=edgenode + cache-to: type=gha,mode=max,scope=edgenode - name: Run tests run: | @@ -46,18 +58,28 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Build log-collector (cached) + uses: docker/build-push-action@v5 + with: + context: docker/log-collector + load: true + tags: edgenode-log-collector:latest + cache-from: type=gha,scope=log-collector + cache-to: type=gha,mode=max,scope=log-collector + - name: Build harvester image for testing + uses: docker/build-push-action@v5 env: GITHUB_TOKEN: ${{ secrets.HARVESTER_REPO_TOKEN }} - run: | - docker buildx build docker/ --file docker/Dockerfile.harvester \ - --platform linux/amd64 \ - --secret id=github_token,env=GITHUB_TOKEN \ - --load \ - --tag local-edgenode-harvester + with: + context: docker/ + file: docker/Dockerfile.harvester + load: true + tags: local-edgenode-harvester + secret-envs: github_token=GITHUB_TOKEN + cache-from: type=gha,scope=edgenode-harvester + cache-to: type=gha,mode=max,scope=edgenode-harvester - name: Run harvester tests - env: - GITHUB_TOKEN: ${{ secrets.HARVESTER_REPO_TOKEN }} run: | CI_RELEASE_TEST=true ./docker/run_harvester_tests.sh local-edgenode-harvester diff --git a/docker/Dockerfile.harvester b/docker/Dockerfile.harvester index a543e3f..e6e6388 100644 --- a/docker/Dockerfile.harvester +++ b/docker/Dockerfile.harvester @@ -1,8 +1,9 @@ # syntax=docker/dockerfile:1 # Dockerfile for the edge node harvester variant. # Replaces log-sender with log-harvester (unytco/log-harvester). -# Requires GITHUB_TOKEN build secret to clone the private log-harvester repo: +# In CI, provide a GITHUB_TOKEN secret to clone the private repo: # docker buildx build --secret id=github_token,env=GITHUB_TOKEN ... +# Locally, omit the secret if your git credentials already grant access. FROM cgr.dev/chainguard/wolfi-base @@ -43,8 +44,13 @@ ARG LOG_HARVESTER_VERSION=main RUN --mount=type=secret,id=github_token \ mkdir -p /app && \ - git clone --depth 1 --branch ${LOG_HARVESTER_VERSION} \ - https://$(cat /run/secrets/github_token)@github.com/unytco/log-harvester.git /app/src && \ + TOKEN="$(cat /run/secrets/github_token 2>/dev/null || true)" && \ + if [ -n "$TOKEN" ]; then \ + REPO_URL="https://${TOKEN}@github.com/unytco/log-harvester.git"; \ + else \ + REPO_URL="https://github.com/unytco/log-harvester.git"; \ + fi && \ + git clone --depth 1 --branch ${LOG_HARVESTER_VERSION} "${REPO_URL}" /app/src && \ cd /app/src && \ npm ci && \ npm run build && \ @@ -98,7 +104,7 @@ RUN chmod +x \ ENV HC_ADMIN_PORT=4444 ENV HC_APP_PORT=4445 ENV HAPP_PATH=/app/unyt.happ -ENV CONFIG_PATH=/etc/log-harvester/config.json +ENV CONFIG_PATH=/data/log-harvester/config.json SHELL ["/bin/sh", "-c"] EXPOSE 4444 4445 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 36a0250..ca26fae 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -62,7 +62,6 @@ services: - RUST_LOG=${RUST_LOG:-info} volumes: - holo-data-harvester:/data - - holo-config-harvester:/etc/log-harvester networks: - test-net healthcheck: @@ -84,5 +83,3 @@ volumes: driver: local holo-data-harvester: driver: local - holo-config-harvester: - driver: local diff --git a/docker/run_tests_multi.sh b/docker/run_tests_multi.sh index 5d7ab2e..cd0f25c 100755 --- a/docker/run_tests_multi.sh +++ b/docker/run_tests_multi.sh @@ -31,9 +31,13 @@ export SERVICE_NAME export SCRIPT_DIR export COMPOSE_PROJECT_NAME="edgenode" -# Start services (--build needed for log-collector) +# Build log-collector separately so Docker layer cache is used on subsequent runs +echo "Building log-collector..." +docker compose build log-collector + +# Start services without rebuilding (log-collector already built above) echo "Starting services..." -docker compose up --build -d +docker compose up -d # Wait for log-collector echo "Waiting for log-collector to be healthy..." diff --git a/docker/s6-overlay-harvester/s6-rc.d/conductor/run b/docker/s6-overlay-harvester/s6-rc.d/conductor/run index 10a9385..e7b53ee 100644 --- a/docker/s6-overlay-harvester/s6-rc.d/conductor/run +++ b/docker/s6-overlay-harvester/s6-rc.d/conductor/run @@ -1,4 +1,4 @@ -#!/bin/sh +#!/command/with-contenv /bin/sh if [ "${CONDUCTOR_MODE:-}" = "false" ]; then exec tail -f /dev/null fi diff --git a/docker/s6-overlay-harvester/s6-rc.d/log-harvester/run b/docker/s6-overlay-harvester/s6-rc.d/log-harvester/run index f8924f1..f0894e6 100644 --- a/docker/s6-overlay-harvester/s6-rc.d/log-harvester/run +++ b/docker/s6-overlay-harvester/s6-rc.d/log-harvester/run @@ -1,4 +1,4 @@ -#!/bin/sh +#!/command/with-contenv /bin/sh set -e APP_ID="${HC_APP_ID:-unyt}" @@ -23,10 +23,6 @@ else echo "WARNING: ${HAPP_PATH} not found — skipping happ install" >> /data/logs/startup.log fi -# Attach app websocket on HC_APP_PORT (idempotent) -hc sandbox call --running "${HC_ADMIN_PORT}" add-app-ws "${HC_APP_PORT}" \ - --allowed-origins "log-harvester" >> /data/logs/startup.log 2>&1 || true - # Wait a moment for keystore to be fully ready before touching credentials sleep 3 @@ -40,6 +36,7 @@ if [ ! -f "${CONFIG_PATH}" ]; then --hc-admin "${HC_ADMIN_PORT}" \ --hc-app "${HC_APP_PORT}" \ --app-id "${APP_ID}" >> /data/logs/startup.log 2>&1 + chown nonroot:nonroot "${CONFIG_PATH}" 2>/dev/null || true else echo "Refreshing harvester credentials..." >> /data/logs/startup.log cp "${CONFIG_PATH}" "${CONFIG_PATH}.backup" @@ -61,6 +58,7 @@ else jq ".lastInvoice = ${LAST_INVOICE}" "${CONFIG_PATH}" > "${CONFIG_PATH}.tmp" && \ mv "${CONFIG_PATH}.tmp" "${CONFIG_PATH}" || true fi + chown nonroot:nonroot "${CONFIG_PATH}" 2>/dev/null || true fi echo "Starting log-harvester service..." >> /data/logs/startup.log diff --git a/docker/s6-overlay-harvester/s6-rc.d/setup/type b/docker/s6-overlay-harvester/s6-rc.d/setup/type new file mode 100644 index 0000000..bdd22a1 --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/setup/type @@ -0,0 +1 @@ +oneshot diff --git a/docker/s6-overlay-harvester/s6-rc.d/setup/up b/docker/s6-overlay-harvester/s6-rc.d/setup/up new file mode 100755 index 0000000..665ea0c --- /dev/null +++ b/docker/s6-overlay-harvester/s6-rc.d/setup/up @@ -0,0 +1,2 @@ +#\!/bin/execlineb -P +/bin/sh /etc/s6-overlay/s6-rc.d/setup/init.sh diff --git a/docker/tests/harvester_startup.bats b/docker/tests/harvester_startup.bats index 069a1cd..b299d14 100644 --- a/docker/tests/harvester_startup.bats +++ b/docker/tests/harvester_startup.bats @@ -17,22 +17,21 @@ HARVESTER_SERVICE="${SERVICE_NAME:-edgenode-harvester}" assert_output --partial "unyt" } -@test "App websocket attached on port 4445" { +@test "log-harvester has connected to app websocket" { run docker compose exec -T "$HARVESTER_SERVICE" \ - sh -c "cat /data/logs/startup.log" + test -s /data/logs/log-harvester.log assert_success - assert_output --partial "add-app-ws" } @test "Harvester config is initialized" { run docker compose exec -T "$HARVESTER_SERVICE" \ - test -f /etc/log-harvester/config.json + test -f /data/log-harvester/config.json assert_success } -@test "Harvester config contains droneId" { +@test "Harvester config contains collectorUrl" { run docker compose exec -T "$HARVESTER_SERVICE" \ - jq -e '.droneId' /etc/log-harvester/config.json + jq -e '.collectorUrl' /data/log-harvester/config.json assert_success } From 82c6b6d17a5ccea8cb614ba53cb0cc3ea37f09c3 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:35:39 +0000 Subject: [PATCH 06/21] ci: clone unytco/log-collector in CI before building The docker/log-collector directory is gitignored (it's a separate repo). Add an actions/checkout step to clone it into place before the build. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 093293b..bf63335 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -23,6 +23,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Checkout log-collector + uses: actions/checkout@v4 + with: + repository: unytco/log-collector + path: docker/log-collector + - name: Build log-collector (cached) uses: docker/build-push-action@v5 with: @@ -58,6 +64,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Checkout log-collector + uses: actions/checkout@v4 + with: + repository: unytco/log-collector + path: docker/log-collector + - name: Build log-collector (cached) uses: docker/build-push-action@v5 with: From 5957d6ff51d4029093717ecc35ae9e40477e503d Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:36:58 +0000 Subject: [PATCH 07/21] ci: authenticate log-collector checkout with HARVESTER_REPO_TOKEN Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index bf63335..6af1acb 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -28,6 +28,7 @@ jobs: with: repository: unytco/log-collector path: docker/log-collector + token: ${{ secrets.HARVESTER_REPO_TOKEN }} - name: Build log-collector (cached) uses: docker/build-push-action@v5 @@ -69,6 +70,7 @@ jobs: with: repository: unytco/log-collector path: docker/log-collector + token: ${{ secrets.HARVESTER_REPO_TOKEN }} - name: Build log-collector (cached) uses: docker/build-push-action@v5 From 791dd38618f12e9c42fcf3b1abf8d8b544b5f509 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:44:33 +0000 Subject: [PATCH 08/21] ci: add Dockerfile.log-collector to repo, reference from compose and CI The log-collector Dockerfile and entrypoint are ours, not upstream's. Store as Dockerfile.log-collector in the docker/ directory so CI can find them after checking out unytco/log-collector as the build context. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 2 ++ docker/Dockerfile.log-collector | 28 ++++++++++++++++++++++++++++ docker/docker-compose.yml | 2 +- 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile.log-collector diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 6af1acb..982ab10 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -34,6 +34,7 @@ jobs: uses: docker/build-push-action@v5 with: context: docker/log-collector + file: docker/Dockerfile.log-collector load: true tags: edgenode-log-collector:latest cache-from: type=gha,scope=log-collector @@ -76,6 +77,7 @@ jobs: uses: docker/build-push-action@v5 with: context: docker/log-collector + file: docker/Dockerfile.log-collector load: true tags: edgenode-log-collector:latest cache-from: type=gha,scope=log-collector diff --git a/docker/Dockerfile.log-collector b/docker/Dockerfile.log-collector new file mode 100644 index 0000000..91e5dd8 --- /dev/null +++ b/docker/Dockerfile.log-collector @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 +# Dockerfile for the log-collector service. +# Build context should be a checkout of unytco/log-collector. + +FROM node:20-slim + +# Install wrangler, curl for health check, and sqlite3 for debugging +RUN npm install -g wrangler@4.45.4 && \ + apt-get update && \ + apt-get install -y curl sqlite3 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Copy worker source from build context (unytco/log-collector checkout) +WORKDIR /usr/src/app +COPY . . + +# Install dependencies +RUN npm install + +EXPOSE 8787 + +RUN printf '#!/bin/bash\nset -e\necho "[log-collector] Starting wrangler dev server..."\nexec "$@"\n' \ + > /usr/local/bin/docker-entrypoint.sh && \ + chmod +x /usr/local/bin/docker-entrypoint.sh + +ENTRYPOINT ["docker-entrypoint.sh"] +CMD ["wrangler", "dev", "--env", "development", "--ip", "0.0.0.0", "--port", "8787"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ca26fae..d758f21 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -28,7 +28,7 @@ services: log-collector: build: context: ./log-collector - dockerfile: Dockerfile + dockerfile: ../Dockerfile.log-collector ports: - "8787:8787" environment: From a7b5e8f986d47a7cd627d97b870cf21e76932de7 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:53:31 +0000 Subject: [PATCH 09/21] =?UTF-8?q?ci:=20fix=20harvester=20build=20secret=20?= =?UTF-8?q?=E2=80=94=20use=20secrets:=20not=20secret-envs:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit secret-envs is not a valid build-push-action parameter; the correct field is secrets which mounts the value directly as a build secret. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 982ab10..60e20ca 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -85,14 +85,13 @@ jobs: - name: Build harvester image for testing uses: docker/build-push-action@v5 - env: - GITHUB_TOKEN: ${{ secrets.HARVESTER_REPO_TOKEN }} with: context: docker/ file: docker/Dockerfile.harvester load: true tags: local-edgenode-harvester - secret-envs: github_token=GITHUB_TOKEN + secrets: | + github_token=${{ secrets.HARVESTER_REPO_TOKEN }} cache-from: type=gha,scope=edgenode-harvester cache-to: type=gha,mode=max,scope=edgenode-harvester From 87e6adb063f5f6bf14658b9ce25a452a948e45c6 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 20:01:22 +0000 Subject: [PATCH 10/21] ci: use run: step for harvester build to fix secret passing build-push-action secrets: does not reliably mount the github_token secret into the BuildKit RUN --mount. Revert to an explicit docker buildx build run: step with --secret id=github_token,env=GITHUB_TOKEN, which is the known-working approach. GHA layer cache flags are retained. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 60e20ca..00e5b97 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -84,16 +84,16 @@ jobs: cache-to: type=gha,mode=max,scope=log-collector - name: Build harvester image for testing - uses: docker/build-push-action@v5 - with: - context: docker/ - file: docker/Dockerfile.harvester - load: true - tags: local-edgenode-harvester - secrets: | - github_token=${{ secrets.HARVESTER_REPO_TOKEN }} - cache-from: type=gha,scope=edgenode-harvester - cache-to: type=gha,mode=max,scope=edgenode-harvester + env: + GITHUB_TOKEN: ${{ secrets.HARVESTER_REPO_TOKEN }} + run: | + docker buildx build docker/ \ + --file docker/Dockerfile.harvester \ + --secret id=github_token,env=GITHUB_TOKEN \ + --cache-from type=gha,scope=edgenode-harvester \ + --cache-to type=gha,mode=max,scope=edgenode-harvester \ + --load \ + --tag local-edgenode-harvester - name: Run harvester tests run: | From 1027b59d76b7eb2ec92c09f2a849e4545b9a2e28 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 20:09:37 +0000 Subject: [PATCH 11/21] fix: pre-clone log-harvester source instead of using Docker build secrets Docker BuildKit secret mounting is unreliable in GHA. Mirror the log-collector approach: checkout unytco/log-harvester into docker/log-harvester-src/ before the build, then COPY it in. - Dockerfile.harvester: replace RUN --mount=type=secret git clone with COPY log-harvester-src - pr-checks.yml: add actions/checkout step for log-harvester - run_harvester_tests.sh: auto-clone log-harvester-src if absent - .gitignore: add docker/log-harvester-src/ Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 25 +++++++++++++++---------- .gitignore | 1 + docker/Dockerfile.harvester | 23 +++++++++-------------- docker/run_harvester_tests.sh | 13 +++++++------ 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 00e5b97..1503852 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -83,17 +83,22 @@ jobs: cache-from: type=gha,scope=log-collector cache-to: type=gha,mode=max,scope=log-collector + - name: Checkout log-harvester + uses: actions/checkout@v4 + with: + repository: unytco/log-harvester + path: docker/log-harvester-src + token: ${{ secrets.HARVESTER_REPO_TOKEN }} + - name: Build harvester image for testing - env: - GITHUB_TOKEN: ${{ secrets.HARVESTER_REPO_TOKEN }} - run: | - docker buildx build docker/ \ - --file docker/Dockerfile.harvester \ - --secret id=github_token,env=GITHUB_TOKEN \ - --cache-from type=gha,scope=edgenode-harvester \ - --cache-to type=gha,mode=max,scope=edgenode-harvester \ - --load \ - --tag local-edgenode-harvester + uses: docker/build-push-action@v5 + with: + context: docker/ + file: docker/Dockerfile.harvester + load: true + tags: local-edgenode-harvester + cache-from: type=gha,scope=edgenode-harvester + cache-to: type=gha,mode=max,scope=edgenode-harvester - name: Run harvester tests run: | diff --git a/.gitignore b/.gitignore index 107c7ec..b3e0ca7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ holo-data-test/ .gemini/ docker/happ_config_file docker/log-collector/ +docker/log-harvester-src/ AGENTS.md log_fix.md UNYT_TEST_ANALYSIS.md diff --git a/docker/Dockerfile.harvester b/docker/Dockerfile.harvester index e6e6388..290cbec 100644 --- a/docker/Dockerfile.harvester +++ b/docker/Dockerfile.harvester @@ -1,9 +1,10 @@ # syntax=docker/dockerfile:1 # Dockerfile for the edge node harvester variant. # Replaces log-sender with log-harvester (unytco/log-harvester). -# In CI, provide a GITHUB_TOKEN secret to clone the private repo: -# docker buildx build --secret id=github_token,env=GITHUB_TOKEN ... -# Locally, omit the secret if your git credentials already grant access. +# Requires log-harvester source at log-harvester-src/ in the build context: +# CI: pre-cloned via actions/checkout (unytco/log-harvester) +# Local: run_harvester_tests.sh clones it automatically, or: +# git clone --depth 1 https://github.com/unytco/log-harvester.git log-harvester-src FROM cgr.dev/chainguard/wolfi-base @@ -40,17 +41,11 @@ RUN case "${TARGETARCH}" in \ chmod +x /bin/holochain && \ chmod +x /bin/hc -ARG LOG_HARVESTER_VERSION=main - -RUN --mount=type=secret,id=github_token \ - mkdir -p /app && \ - TOKEN="$(cat /run/secrets/github_token 2>/dev/null || true)" && \ - if [ -n "$TOKEN" ]; then \ - REPO_URL="https://${TOKEN}@github.com/unytco/log-harvester.git"; \ - else \ - REPO_URL="https://github.com/unytco/log-harvester.git"; \ - fi && \ - git clone --depth 1 --branch ${LOG_HARVESTER_VERSION} "${REPO_URL}" /app/src && \ +# log-harvester source must be present at log-harvester-src/ in the build context. +# CI: pre-cloned via actions/checkout (unytco/log-harvester) +# Local: git clone --depth 1 https://github.com/unytco/log-harvester.git log-harvester-src +COPY log-harvester-src /app/src +RUN mkdir -p /app && \ cd /app/src && \ npm ci && \ npm run build && \ diff --git a/docker/run_harvester_tests.sh b/docker/run_harvester_tests.sh index 9c30bac..7764e48 100755 --- a/docker/run_harvester_tests.sh +++ b/docker/run_harvester_tests.sh @@ -25,14 +25,15 @@ if [[ "$CI_RELEASE_TEST" != "true" ]] && [[ "$IMAGE_NAME" == local-edgenode-harv if [[ "$FORCE_REBUILD" != "true" ]] && docker image inspect "$IMAGE_NAME" >/dev/null 2>&1; then echo "Using existing local image: $IMAGE_NAME (set FORCE_REBUILD=true to rebuild)" else - echo "Building local harvester image: $IMAGE_NAME" - SECRET_ARGS="" - if [ -n "$GITHUB_TOKEN" ]; then - SECRET_ARGS="--secret id=github_token,env=GITHUB_TOKEN" + # Ensure log-harvester source is present in the build context + if [ ! -d "$SCRIPT_DIR/log-harvester-src/.git" ]; then + echo "Cloning log-harvester source..." + git clone --depth 1 \ + https://github.com/unytco/log-harvester.git \ + "$SCRIPT_DIR/log-harvester-src" fi - # shellcheck disable=SC2086 + echo "Building local harvester image: $IMAGE_NAME" docker build \ - $SECRET_ARGS \ -t "$IMAGE_NAME" \ -f Dockerfile.harvester \ . From b12d5a6a061896c10fb0efbe5de3e24c61efd7b2 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 20:13:30 +0000 Subject: [PATCH 12/21] fix: start only edgenode+log-collector in run_tests_multi.sh docker compose up -d was starting all services including edgenode-harvester, which requires log-harvester-src in the build context. Explicitly name only the services needed for the edgenode test suite. Co-Authored-By: Claude Sonnet 4.6 --- docker/run_tests_multi.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/run_tests_multi.sh b/docker/run_tests_multi.sh index cd0f25c..c23c096 100755 --- a/docker/run_tests_multi.sh +++ b/docker/run_tests_multi.sh @@ -35,9 +35,9 @@ export COMPOSE_PROJECT_NAME="edgenode" echo "Building log-collector..." docker compose build log-collector -# Start services without rebuilding (log-collector already built above) +# Start only the services needed for edgenode tests (not edgenode-harvester) echo "Starting services..." -docker compose up -d +docker compose up -d log-collector "$SERVICE_NAME" # Wait for log-collector echo "Waiting for log-collector to be healthy..." From e7607811f8972f8851a804c3cce0ad8c232a152f Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 20:28:04 +0000 Subject: [PATCH 13/21] fix: apply D1 schema on log-collector startup In CI the wrangler state directory is empty (no persistent local volume), so drone_registrations and other tables don't exist. Run wrangler d1 execute --local --file=schema.sql in the entrypoint before starting wrangler dev. Co-Authored-By: Claude Sonnet 4.6 --- docker/Dockerfile.log-collector | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.log-collector b/docker/Dockerfile.log-collector index 91e5dd8..26d9a75 100644 --- a/docker/Dockerfile.log-collector +++ b/docker/Dockerfile.log-collector @@ -20,7 +20,7 @@ RUN npm install EXPOSE 8787 -RUN printf '#!/bin/bash\nset -e\necho "[log-collector] Starting wrangler dev server..."\nexec "$@"\n' \ +RUN printf '#!/bin/bash\nset -e\necho "[log-collector] Applying D1 schema..."\nnpx wrangler d1 execute log-collector-db --local --file=schema.sql 2>&1 || true\necho "[log-collector] Starting wrangler dev server..."\nexec "$@"\n' \ > /usr/local/bin/docker-entrypoint.sh && \ chmod +x /usr/local/bin/docker-entrypoint.sh From edf3e97bb2fb8bcc362b59c633ef5c6a304e6feb Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 20:44:01 +0000 Subject: [PATCH 14/21] fix: pass ADMIN_SECRET to wrangler dev via --var Wrangler dev reads vars from wrangler.toml, not Docker env vars. Pass --var ADMIN_SECRET:${ADMIN_SECRET} so the worker picks up the secret configured in docker-compose.yml. Co-Authored-By: Claude Sonnet 4.6 --- docker/Dockerfile.log-collector | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile.log-collector b/docker/Dockerfile.log-collector index 26d9a75..3c4f780 100644 --- a/docker/Dockerfile.log-collector +++ b/docker/Dockerfile.log-collector @@ -20,9 +20,8 @@ RUN npm install EXPOSE 8787 -RUN printf '#!/bin/bash\nset -e\necho "[log-collector] Applying D1 schema..."\nnpx wrangler d1 execute log-collector-db --local --file=schema.sql 2>&1 || true\necho "[log-collector] Starting wrangler dev server..."\nexec "$@"\n' \ +RUN printf '#!/bin/bash\nset -e\necho "[log-collector] Applying D1 schema..."\nnpx wrangler d1 execute log-collector-db --local --file=schema.sql 2>&1 || true\necho "[log-collector] Starting wrangler dev server..."\nexec wrangler dev --env development --ip 0.0.0.0 --port 8787 \\\n --var ADMIN_SECRET:"${ADMIN_SECRET:-test_admin_secret}"\n' \ > /usr/local/bin/docker-entrypoint.sh && \ chmod +x /usr/local/bin/docker-entrypoint.sh ENTRYPOINT ["docker-entrypoint.sh"] -CMD ["wrangler", "dev", "--env", "development", "--ip", "0.0.0.0", "--port", "8787"] From 00fc84219d24b4fb097df4c777a854380cd11882 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 21:02:39 +0000 Subject: [PATCH 15/21] fix: exclude harvester tests from run_tests_multi.sh; add setup guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_tests_multi.sh was running all *.bats files including harvester_*, which fail when edgenode-harvester isn't started. Exclude harvester files explicitly — they belong to run_harvester_tests.sh. Also add setup() skip guards to harvester_startup.bats and harvester_process.bats which were missing them. Co-Authored-By: Claude Sonnet 4.6 --- docker/run_tests_multi.sh | 4 ++-- docker/tests/harvester_process.bats | 6 ++++++ docker/tests/harvester_startup.bats | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/run_tests_multi.sh b/docker/run_tests_multi.sh index c23c096..8428387 100755 --- a/docker/run_tests_multi.sh +++ b/docker/run_tests_multi.sh @@ -66,10 +66,10 @@ else echo "Warning: container not found, using fallback: $CONTAINER_NAME" fi -# Run tests +# Run edgenode tests — exclude harvester-specific files (those run via run_harvester_tests.sh) echo "Running tests..." set +e -./tests/libs/bats/bin/bats tests +./tests/libs/bats/bin/bats $(find tests -maxdepth 1 -name '*.bats' ! -name 'harvester_*' | sort) TEST_EXIT_CODE=$? set -e diff --git a/docker/tests/harvester_process.bats b/docker/tests/harvester_process.bats index d1e13cd..d48ee96 100644 --- a/docker/tests/harvester_process.bats +++ b/docker/tests/harvester_process.bats @@ -5,6 +5,12 @@ load 'libs/bats-assert/load' HARVESTER_SERVICE="${SERVICE_NAME:-edgenode-harvester}" +setup() { + if ! docker compose ps "$HARVESTER_SERVICE" 2>/dev/null | grep -q "running\|Up"; then + skip "edgenode-harvester service is not running" + fi +} + @test "Holochain process runs as nonroot" { run docker compose exec -T "$HARVESTER_SERVICE" \ sh -c "ps aux | grep -E 'nonroot.*holochain'" diff --git a/docker/tests/harvester_startup.bats b/docker/tests/harvester_startup.bats index b299d14..7eb25fb 100644 --- a/docker/tests/harvester_startup.bats +++ b/docker/tests/harvester_startup.bats @@ -5,6 +5,12 @@ load 'libs/bats-assert/load' HARVESTER_SERVICE="${SERVICE_NAME:-edgenode-harvester}" +setup() { + if ! docker compose ps "$HARVESTER_SERVICE" 2>/dev/null | grep -q "running\|Up"; then + skip "edgenode-harvester service is not running" + fi +} + @test "Conductor starts successfully" { run docker compose logs "$HARVESTER_SERVICE" assert_output --partial "Conductor ready." From 864a3c39740abe96eb6e1434d75ef5035efb152c Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 21:22:24 +0000 Subject: [PATCH 16/21] fix: prevent wrangler crash cascade in edgenode test suite - Add restart: unless-stopped to log-collector service so Docker auto-recovers when wrangler dev crashes under concurrent D1 load - Tighten healthcheck (15s interval, 5 retries, 30s start_period) - Run each .bats file separately in run_tests_multi.sh with a log-collector health-check wait between files, so a wrangler crash in integration_data_pipeline.bats doesn't cascade to later files Co-Authored-By: Claude Sonnet 4.6 --- docker/docker-compose.yml | 6 ++++-- docker/run_tests_multi.sh | 30 ++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d758f21..4cd0e4a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -36,11 +36,13 @@ services: - ENVIRONMENT=${ENVIRONMENT:-development} networks: - test-net + restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8787"] - interval: 30s + interval: 15s timeout: 10s - retries: 3 + retries: 5 + start_period: 30s edgenode-harvester: image: ${HARVESTER_IMAGE:-local-edgenode-harvester} diff --git a/docker/run_tests_multi.sh b/docker/run_tests_multi.sh index 8428387..1756ec7 100755 --- a/docker/run_tests_multi.sh +++ b/docker/run_tests_multi.sh @@ -66,11 +66,37 @@ else echo "Warning: container not found, using fallback: $CONTAINER_NAME" fi +# Wait for log-collector to be healthy (with restart support) +wait_for_log_collector() { + local max_wait="${1:-60}" + local elapsed=0 + while [ $elapsed -lt $max_wait ]; do + if curl -sf http://localhost:8787/ >/dev/null 2>&1; then + return 0 + fi + sleep 3 + elapsed=$((elapsed + 3)) + done + echo "Warning: log-collector not healthy after ${max_wait}s, proceeding anyway" + return 0 +} + # Run edgenode tests — exclude harvester-specific files (those run via run_harvester_tests.sh) +# Run each file separately so a wrangler crash in one file doesn't cascade to subsequent files. echo "Running tests..." +TEST_EXIT_CODE=0 set +e -./tests/libs/bats/bin/bats $(find tests -maxdepth 1 -name '*.bats' ! -name 'harvester_*' | sort) -TEST_EXIT_CODE=$? +for BATS_FILE in $(find tests -maxdepth 1 -name '*.bats' ! -name 'harvester_*' | sort); do + echo "--- Waiting for log-collector before: $BATS_FILE ---" + wait_for_log_collector 60 + echo "--- Running: $BATS_FILE ---" + ./tests/libs/bats/bin/bats "$BATS_FILE" + FILE_EXIT=$? + if [ $FILE_EXIT -ne 0 ]; then + echo "FAILED: $BATS_FILE (exit $FILE_EXIT)" + TEST_EXIT_CODE=$FILE_EXIT + fi +done set -e if [ $TEST_EXIT_CODE -ne 0 ]; then From 94cc3d7def0b6751d13b533a99ea8b064f1e10c2 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 22:49:37 +0000 Subject: [PATCH 17/21] docs: update CHANGELOG, TESTING, and quickstart for harvester additions - CHANGELOG: add all Unreleased entries for harvester variant work (Dockerfile.log-collector, run_harvester_tests.sh, BATS files, test runner isolation, restart policy) - TESTING: note per-file test isolation and harvester_* naming convention - LOG_HARVESTER_QUICKSTART: remove stale --secret flag from local build snippet; replace with git clone of log-harvester-src (required for COPY) Co-Authored-By: Claude Sonnet 4.6 --- docker/CHANGELOG.md | 8 ++++++++ docker/LOG_HARVESTER_QUICKSTART.md | 4 +++- docker/TESTING.md | 4 +++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/CHANGELOG.md b/docker/CHANGELOG.md index 1b79718..dcb12d0 100644 --- a/docker/CHANGELOG.md +++ b/docker/CHANGELOG.md @@ -12,8 +12,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `s6-overlay-harvester/` service tree: self-contained s6 services for the harvester variant (`conductor`, `log-harvester`, `logrotate-cron`, `setup`) - `log-harvester` s6 longrun service: waits for conductor, installs `unyt.happ`, attaches app websocket on port 4445, initializes/refreshes harvester config, runs harvester loop - `unyt.happ` baked into the harvester image from latest `unytco/unyt-sandbox` release; pinnable via `UNYT_HAPP_VERSION` build arg +- `Dockerfile.log-collector` — Dockerfile for the `unytco/log-collector` Cloudflare Worker service; applies D1 schema on startup and passes `ADMIN_SECRET` via wrangler `--var` +- `run_harvester_tests.sh` — dedicated test runner for the harvester variant; auto-clones `log-harvester-src` if not present, starts all three services, waits for readiness +- Harvester BATS test suite: `harvester_startup.bats`, `harvester_process.bats`, `harvester_integration.bats`, `harvester_e2e.bats` - `LOG_HARVESTER_QUICKSTART.md` quickstart guide for the harvester variant - CI `build-and-push-harvester-image` job in release workflow publishing `ghcr.io/holo-host/edgenode-harvester` +- CI PR checks now build and test both `edgenode` and `edgenode-harvester` images with GHA layer caching + +### Changed +- `run_tests_multi.sh` runs each `.bats` file individually with a log-collector health-check between files, preventing wrangler crash cascades from affecting subsequent test files +- `docker-compose.yml`: log-collector service gets `restart: unless-stopped` so Docker auto-recovers after wrangler dev crashes under concurrent D1 load ## [0.1.0-alpha1] - 2026-03-13 diff --git a/docker/LOG_HARVESTER_QUICKSTART.md b/docker/LOG_HARVESTER_QUICKSTART.md index 872cbf1..57d5837 100644 --- a/docker/LOG_HARVESTER_QUICKSTART.md +++ b/docker/LOG_HARVESTER_QUICKSTART.md @@ -46,9 +46,11 @@ Images are available from [GitHub Packages](https://github.com/Holo-Host/edgenod The `unyt.happ` is baked in from the latest [unytco/unyt-sandbox](https://github.com/unytco/unyt-sandbox) release. To pin a specific version, build locally: ```bash +# Clone log-harvester source first (required for the COPY step) +git clone --depth 1 https://github.com/unytco/log-harvester.git docker/log-harvester-src + docker buildx build docker/ --file docker/Dockerfile.harvester \ --build-arg UNYT_HAPP_VERSION=v0.62.0 \ - --secret id=github_token,env=GITHUB_TOKEN \ --tag my-edgenode-harvester \ --load ``` diff --git a/docker/TESTING.md b/docker/TESTING.md index 01b7246..1fa096e 100644 --- a/docker/TESTING.md +++ b/docker/TESTING.md @@ -37,7 +37,9 @@ The test cases are in the `tests/` directory: ## Adding New Tests -Create a new `.bats` file in the `tests/` directory. It will be picked up automatically by `run_tests_multi.sh`. +Create a new `.bats` file in the `tests/` directory. It will be picked up automatically by `run_tests_multi.sh`, which runs each file individually with a log-collector health-check between files (so a crash in one file doesn't cascade to the next). + +**Naming convention:** harvester-specific tests must be named `harvester_*.bats`. These are excluded from the edgenode test run and are instead run by `run_harvester_tests.sh`. ## CI Integration From 631796463a278ddf9e0eb277b93b42be782c1b1c Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Thu, 19 Mar 2026 22:50:51 +0000 Subject: [PATCH 18/21] docs: fix stale Dockerfile.unyt and latest-unyt references Dockerfile was renamed/consolidated; latest-unyt tag is not published. Co-Authored-By: Claude Sonnet 4.6 --- docker/LOG_SENDER_QUICKSTART.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/LOG_SENDER_QUICKSTART.md b/docker/LOG_SENDER_QUICKSTART.md index 41d542d..8acadcd 100644 --- a/docker/LOG_SENDER_QUICKSTART.md +++ b/docker/LOG_SENDER_QUICKSTART.md @@ -2,7 +2,7 @@ Connect an Edge Node to a Unyt log-collector for resource accounting. -See the [Docker README.md](./README.md) for basic Edge Node setup. The unyt image is built from [Dockerfile.unyt](./Dockerfile.unyt). +See the [Docker README.md](./README.md) for basic Edge Node setup. The edge node image is built from [Dockerfile](./Dockerfile). For the upstream log-sender docs, see the [Log-Sender User Guide](https://github.com/unytco/log-sender/blob/main/LOG_SENDER_USER_GUIDE.md). @@ -25,7 +25,7 @@ docker run --name unytnode -dit \ -p 4444:4444 \ -e LOG_SENDER_ENDPOINT=http://your-log-collector:8787 \ -e LOG_SENDER_UNYT_PUB_KEY=uhCAk... \ - ghcr.io/holo-host/edgenode:latest-unyt + ghcr.io/holo-host/edgenode ``` Wait for the conductor to start: From 0ce7d56c3040bc8aadf34d6b0e56566804d7cf26 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:00:19 +0100 Subject: [PATCH 19/21] fix: resolve review issues in harvester CI and compose config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - release.yml: add missing Checkout log-harvester step before build; remove dead --secret flag (Dockerfile now uses COPY, not build secret) - pr-checks.yml: remove log-collector checkout from test-docker-image job — only needed by test-harvester-image; avoids coupling standard edgenode tests to HARVESTER_REPO_TOKEN - docker-compose.yml: remove dead build secrets block on edgenode-harvester and top-level secrets declaration (Dockerfile no longer uses build secret) - Dockerfile.harvester: remove redundant mkdir -p /app (COPY creates the dir) Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 17 ----------------- .github/workflows/release.yml | 12 ++++++++---- docker/Dockerfile.harvester | 3 +-- docker/docker-compose.yml | 6 ------ 4 files changed, 9 insertions(+), 29 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 1503852..6b9ebfd 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -23,23 +23,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Checkout log-collector - uses: actions/checkout@v4 - with: - repository: unytco/log-collector - path: docker/log-collector - token: ${{ secrets.HARVESTER_REPO_TOKEN }} - - - name: Build log-collector (cached) - uses: docker/build-push-action@v5 - with: - context: docker/log-collector - file: docker/Dockerfile.log-collector - load: true - tags: edgenode-log-collector:latest - cache-from: type=gha,scope=log-collector - cache-to: type=gha,mode=max,scope=log-collector - - name: Build image for testing uses: docker/build-push-action@v5 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5275a76..4c474e5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -176,6 +176,13 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Checkout log-harvester + uses: actions/checkout@v4 + with: + repository: unytco/log-harvester + path: docker/log-harvester-src + token: ${{ secrets.HARVESTER_REPO_TOKEN }} + - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: @@ -192,7 +199,4 @@ jobs: --platform linux/amd64,linux/arm64 \ --push \ --tag "${VERSION_TAG}" \ - --tag "${LATEST_TAG}" \ - --secret id=github_token,env=GITHUB_TOKEN - env: - GITHUB_TOKEN: ${{ secrets.HARVESTER_REPO_TOKEN }} + --tag "${LATEST_TAG}" diff --git a/docker/Dockerfile.harvester b/docker/Dockerfile.harvester index 290cbec..47ae912 100644 --- a/docker/Dockerfile.harvester +++ b/docker/Dockerfile.harvester @@ -45,8 +45,7 @@ RUN case "${TARGETARCH}" in \ # CI: pre-cloned via actions/checkout (unytco/log-harvester) # Local: git clone --depth 1 https://github.com/unytco/log-harvester.git log-harvester-src COPY log-harvester-src /app/src -RUN mkdir -p /app && \ - cd /app/src && \ +RUN cd /app/src && \ npm ci && \ npm run build && \ cp -r dist /app/dist && \ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4cd0e4a..1670b6d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -49,8 +49,6 @@ services: build: context: . dockerfile: Dockerfile.harvester - secrets: - - github_token depends_on: log-collector: condition: service_healthy @@ -76,10 +74,6 @@ services: networks: test-net: -secrets: - github_token: - environment: GITHUB_TOKEN - volumes: holo-data-test: driver: local From 1aaf95cc51069c2cea6f4f41e722aef5bc78a6e5 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:03:03 +0100 Subject: [PATCH 20/21] fix: use GITHUB_TOKEN for log-harvester clone with clear error on failure Mirrors the CI token-authenticated approach. Falls back to unauthenticated clone if GITHUB_TOKEN is unset, but exits with an actionable error message if the clone fails rather than silently producing a broken build context. Co-Authored-By: Claude Sonnet 4.6 --- docker/run_harvester_tests.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/run_harvester_tests.sh b/docker/run_harvester_tests.sh index 7764e48..3854bb4 100755 --- a/docker/run_harvester_tests.sh +++ b/docker/run_harvester_tests.sh @@ -28,9 +28,16 @@ if [[ "$CI_RELEASE_TEST" != "true" ]] && [[ "$IMAGE_NAME" == local-edgenode-harv # Ensure log-harvester source is present in the build context if [ ! -d "$SCRIPT_DIR/log-harvester-src/.git" ]; then echo "Cloning log-harvester source..." - git clone --depth 1 \ - https://github.com/unytco/log-harvester.git \ - "$SCRIPT_DIR/log-harvester-src" + CLONE_URL="https://github.com/unytco/log-harvester.git" + if [ -n "$GITHUB_TOKEN" ]; then + CLONE_URL="https://${GITHUB_TOKEN}@github.com/unytco/log-harvester.git" + fi + git clone --depth 1 "$CLONE_URL" "$SCRIPT_DIR/log-harvester-src" || { + echo "" + echo "Error: failed to clone unytco/log-harvester (private repo)." + echo "Set GITHUB_TOKEN to a PAT with repo read access and retry." + exit 1 + } fi echo "Building local harvester image: $IMAGE_NAME" docker build \ From e1b42adc2dccb5593acb0bcbea4d4cf5b8d2ba66 Mon Sep 17 00:00:00 2001 From: evangineer <53523+evangineer@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:07:46 +0100 Subject: [PATCH 21/21] fix: restore log-collector checkout in test-docker-image job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_tests_multi.sh needs the log-collector source to build and start the service. Removing the checkout broke the Run tests step. The HARVESTER_REPO_TOKEN coupling remains — decoupling would require restructuring run_tests_multi.sh to make log-collector optional. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/pr-checks.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 6b9ebfd..1503852 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -23,6 +23,23 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Checkout log-collector + uses: actions/checkout@v4 + with: + repository: unytco/log-collector + path: docker/log-collector + token: ${{ secrets.HARVESTER_REPO_TOKEN }} + + - name: Build log-collector (cached) + uses: docker/build-push-action@v5 + with: + context: docker/log-collector + file: docker/Dockerfile.log-collector + load: true + tags: edgenode-log-collector:latest + cache-from: type=gha,scope=log-collector + cache-to: type=gha,mode=max,scope=log-collector + - name: Build image for testing uses: docker/build-push-action@v5 with: