From 11c9b5a696bcf9738400bb9216f6e7b81411d276 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Wed, 5 Aug 2026 22:23:28 -0400
Subject: [PATCH 01/21] Add docker compose files

---
 taskfiles/docker.yaml                         |  17 +++
 taskfiles/lint.yaml                           |   3 +-
 tools/deployment/spider-compose/compose.yaml  | 137 ++++++++++++++++++
 .../spider-compose/config/scheduler.yaml      |  21 +++
 .../spider-compose/config/storage.yaml        |  12 ++
 .../spider-compose/config/worker.yaml         |  16 ++
 6 files changed, 205 insertions(+), 1 deletion(-)
 create mode 100644 tools/deployment/spider-compose/compose.yaml
 create mode 100644 tools/deployment/spider-compose/config/scheduler.yaml
 create mode 100644 tools/deployment/spider-compose/config/storage.yaml
 create mode 100644 tools/deployment/spider-compose/config/worker.yaml

diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml
index d08e457fc..d528a749e 100644
--- a/taskfiles/docker.yaml
+++ b/taskfiles/docker.yaml
@@ -2,6 +2,7 @@ version: "3"
 
 vars:
   G_BUILD_SCRIPT: "{{.ROOT_DIR}}/tools/docker/build.sh"
+  G_SPIDER_COMPOSE_DIR: "{{.ROOT_DIR}}/tools/deployment/spider-compose"
 
 tasks:
   build:
@@ -21,3 +22,19 @@ tasks:
   build-worker:
     deps: [":init"]
     cmd: "'{{.G_BUILD_SCRIPT}}' worker"
+
+  compose:validate:
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: "docker compose --file compose.yaml config --quiet"
+
+  compose:up:
+    vars:
+      WAIT_TIMEOUT: "{{.SPIDER_COMPOSE_WAIT_TIMEOUT | default \"120\"}}"
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: >-
+      docker compose --file compose.yaml up --detach --wait
+      --wait-timeout "{{.WAIT_TIMEOUT}}"
+
+  compose:down:
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: "docker compose --file compose.yaml down"
diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml
index aeacb4c5c..c3cd0b776 100644
--- a/taskfiles/lint.yaml
+++ b/taskfiles/lint.yaml
@@ -187,7 +187,8 @@ tasks:
           .github/ \
           docs/ \
           taskfile.yaml \
-          taskfiles/
+          taskfiles/ \
+          tools/deployment/spider-compose/
 
   toml-check:
     cmds:
diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
new file mode 100644
index 000000000..5b2167496
--- /dev/null
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -0,0 +1,137 @@
+# Required variables:
+# - SPIDER_IMAGE_TAG
+# - SPIDER_STORAGE_DB_USERNAME
+# - SPIDER_STORAGE_DB_PASSWORD
+# - SPIDER_DATABASE_ROOT_PASSWORD
+name: "spider"
+
+x-service-defaults: &service_defaults
+  logging:
+    driver: "local"
+  restart: "${SPIDER_RESTART_POLICY:-on-failure:3}"
+  stop_grace_period: "60s"
+
+x-healthcheck-defaults: &healthcheck_defaults
+  interval: "30s"
+  retries: 3
+  start_interval: "2s"
+  start_period: "60s"
+  timeout: "2s"
+
+services:
+  spider-database:
+    <<: *service_defaults
+    image: "mariadb:10.11.16"
+    pull_policy: "always"
+    environment:
+      MYSQL_DATABASE: "spider-db"
+      MYSQL_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?Please set a value.}"
+      MYSQL_ROOT_PASSWORD: "${SPIDER_DATABASE_ROOT_PASSWORD:?Please set a value.}"
+      MYSQL_USER: "${SPIDER_STORAGE_DB_USERNAME:?Please set a value.}"
+    volumes:
+      - type: "volume"
+        source: "spider-database-data"
+        target: "/var/lib/mysql"
+    healthcheck:
+      <<: *healthcheck_defaults
+      test: [
+        "CMD",
+        "mysqladmin", "ping",
+        "--silent",
+        "-h", "127.0.0.1",
+        "-u", "${SPIDER_STORAGE_DB_USERNAME:?Please set a value.}",
+        "--password=${SPIDER_STORAGE_DB_PASSWORD:?Please set a value.}",
+      ]
+
+  spider-storage:
+    <<: *service_defaults
+    image: >-
+      ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:?Please set a value.}
+    pull_policy: "always"
+    command: [
+      "spider-storage",
+      "--config", "/etc/spider/storage.yaml",
+    ]
+    environment:
+      RUST_LOG: "${SPIDER_STORAGE_RUST_LOG:-INFO}"
+      SPIDER_STORAGE_DB_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?Please set a value.}"
+      SPIDER_STORAGE_DB_USERNAME: "${SPIDER_STORAGE_DB_USERNAME:?Please set a value.}"
+    configs:
+      - source: "spider-storage-config"
+        target: "/etc/spider/storage.yaml"
+    depends_on:
+      spider-database:
+        condition: "service_healthy"
+        restart: true
+    healthcheck:
+      <<: *healthcheck_defaults
+      test: [
+        "CMD",
+        "timeout", "2",
+        "bash", "-c", "</dev/tcp/127.0.0.1/50051",
+      ]
+
+  spider-scheduler:
+    <<: *service_defaults
+    image: >-
+      ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:?Please set a value.}
+    pull_policy: "always"
+    command: [
+      "spider-scheduler",
+      "--config", "/etc/spider/scheduler.yaml",
+    ]
+    environment:
+      RUST_LOG: "${SPIDER_SCHEDULER_RUST_LOG:-INFO}"
+    ports:
+      - host_ip: "${SPIDER_SCHEDULER_HOST:-127.0.0.1}"
+        published: "${SPIDER_SCHEDULER_HOST_PORT:-50052}"
+        target: 50052
+    configs:
+      - source: "spider-scheduler-config"
+        target: "/etc/spider/scheduler.yaml"
+    depends_on:
+      spider-storage:
+        condition: "service_healthy"
+        restart: true
+    healthcheck:
+      <<: *healthcheck_defaults
+      test: [
+        "CMD",
+        "timeout", "2",
+        "bash", "-c", "</dev/tcp/127.0.0.1/50052",
+      ]
+
+  spider-worker:
+    <<: *service_defaults
+    image: >-
+      ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:?Please set a value.}
+    pull_policy: "always"
+    command: [
+      "spider-execution-manager",
+      "--config", "/etc/spider/worker.yaml",
+    ]
+    environment:
+      RUST_LOG: "${SPIDER_WORKER_RUST_LOG:-INFO}"
+    configs:
+      - source: "spider-worker-config"
+        target: "/etc/spider/worker.yaml"
+    depends_on:
+      spider-scheduler:
+        condition: "service_healthy"
+        restart: true
+      spider-storage:
+        condition: "service_healthy"
+        restart: true
+    deploy:
+      replicas: "${SPIDER_WORKER_REPLICAS:-4}"
+
+configs:
+  spider-scheduler-config:
+    file: "./config/scheduler.yaml"
+  spider-storage-config:
+    file: "./config/storage.yaml"
+  spider-worker-config:
+    file: "./config/worker.yaml"
+
+volumes:
+  spider-database-data:
diff --git a/tools/deployment/spider-compose/config/scheduler.yaml b/tools/deployment/spider-compose/config/scheduler.yaml
new file mode 100644
index 000000000..6989a0e19
--- /dev/null
+++ b/tools/deployment/spider-compose/config/scheduler.yaml
@@ -0,0 +1,21 @@
+connection_pool_size: 4
+host: "0.0.0.0"
+port: 50052
+runtime:
+  advertised_endpoint:
+    host: "spider-scheduler"
+    port: 50052
+  scheduler:
+    policy: "round_robin"
+    config:
+      active_job_queue_capacity: 64
+      cleanup_ready_task_capacity: 256
+      commit_ready_task_capacity: 256
+      dispatch_queue_capacity: 64
+      finalizing_job_expiration_timeout_sec: 300
+      ready_task_capacity: 65536
+      storage_poll_timeout_ms: 10
+      tick_interval_ms: 5
+storage_endpoint:
+  host: "spider-storage"
+  port: 50051
diff --git a/tools/deployment/spider-compose/config/storage.yaml b/tools/deployment/spider-compose/config/storage.yaml
new file mode 100644
index 000000000..70fef631b
--- /dev/null
+++ b/tools/deployment/spider-compose/config/storage.yaml
@@ -0,0 +1,12 @@
+host: "0.0.0.0"
+port: 50051
+runtime:
+  db_config:
+    host: "spider-database"
+    max_connections: 64
+    name: "spider-db"
+    port: 3306
+  ready_queue_config:
+    cleanup_capacity: 256
+    commit_capacity: 256
+    task_capacity: 1048576
diff --git a/tools/deployment/spider-compose/config/worker.yaml b/tools/deployment/spider-compose/config/worker.yaml
new file mode 100644
index 000000000..79faa5916
--- /dev/null
+++ b/tools/deployment/spider-compose/config/worker.yaml
@@ -0,0 +1,16 @@
+connection_pool_size: 4
+liveness:
+  scheduler_heartbeat_interval_sec: 10
+  storage_heartbeat_interval_sec: 10
+scheduler:
+  host: "spider-scheduler"
+  port: 50052
+scheduler_poll_wait_ms: 1000
+storage:
+  host: "spider-storage"
+  port: 50051
+task_executor:
+  bin_path: "/usr/local/bin/spider-task-executor"
+  inherited_env: []
+  max_log_line_bytes: 65536
+  package_dir: "/opt/spider/packages"

From 470f7e5d69e612b6eba8d160b518633e3fe84594 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Wed, 5 Aug 2026 22:42:20 -0400
Subject: [PATCH 02/21] Fix network

---
 tools/deployment/spider-compose/compose.yaml | 23 ++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
index 5b2167496..62c16c1c8 100644
--- a/tools/deployment/spider-compose/compose.yaml
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -23,6 +23,8 @@ services:
     <<: *service_defaults
     image: "mariadb:10.11.16"
     pull_policy: "always"
+    networks:
+      - "spider-internal"
     environment:
       MYSQL_DATABASE: "spider-db"
       MYSQL_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?Please set a value.}"
@@ -48,6 +50,9 @@ services:
     image: >-
       ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:?Please set a value.}
     pull_policy: "always"
+    networks:
+      - "default"
+      - "spider-internal"
     command: [
       "spider-storage",
       "--config", "/etc/spider/storage.yaml",
@@ -56,6 +61,10 @@ services:
       RUST_LOG: "${SPIDER_STORAGE_RUST_LOG:-INFO}"
       SPIDER_STORAGE_DB_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?Please set a value.}"
       SPIDER_STORAGE_DB_USERNAME: "${SPIDER_STORAGE_DB_USERNAME:?Please set a value.}"
+    ports:
+      - host_ip: "${SPIDER_STORAGE_HOST:-127.0.0.1}"
+        published: "${SPIDER_STORAGE_HOST_PORT:-50051}"
+        target: 50051
     configs:
       - source: "spider-storage-config"
         target: "/etc/spider/storage.yaml"
@@ -76,16 +85,14 @@ services:
     image: >-
       ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:?Please set a value.}
     pull_policy: "always"
+    networks:
+      - "spider-internal"
     command: [
       "spider-scheduler",
       "--config", "/etc/spider/scheduler.yaml",
     ]
     environment:
       RUST_LOG: "${SPIDER_SCHEDULER_RUST_LOG:-INFO}"
-    ports:
-      - host_ip: "${SPIDER_SCHEDULER_HOST:-127.0.0.1}"
-        published: "${SPIDER_SCHEDULER_HOST_PORT:-50052}"
-        target: 50052
     configs:
       - source: "spider-scheduler-config"
         target: "/etc/spider/scheduler.yaml"
@@ -106,6 +113,9 @@ services:
     image: >-
       ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:?Please set a value.}
     pull_policy: "always"
+    networks:
+      - "default"
+      - "spider-internal"
     command: [
       "spider-execution-manager",
       "--config", "/etc/spider/worker.yaml",
@@ -133,5 +143,10 @@ configs:
   spider-worker-config:
     file: "./config/worker.yaml"
 
+networks:
+  default:
+  spider-internal:
+    internal: true
+
 volumes:
   spider-database-data:

From 139286ea4f11edf3afdf07456efbbd000e0a3f32 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Wed, 5 Aug 2026 23:04:50 -0400
Subject: [PATCH 03/21] Add image tag and worker image name

---
 tools/deployment/spider-compose/compose.yaml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
index 62c16c1c8..6bffebdf1 100644
--- a/tools/deployment/spider-compose/compose.yaml
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -1,5 +1,4 @@
 # Required variables:
-# - SPIDER_IMAGE_TAG
 # - SPIDER_STORAGE_DB_USERNAME
 # - SPIDER_STORAGE_DB_PASSWORD
 # - SPIDER_DATABASE_ROOT_PASSWORD
@@ -47,8 +46,7 @@ services:
 
   spider-storage:
     <<: *service_defaults
-    image: >-
-      ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:?Please set a value.}
+    image: "ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:-main}"
     pull_policy: "always"
     networks:
       - "default"
@@ -82,8 +80,7 @@ services:
 
   spider-scheduler:
     <<: *service_defaults
-    image: >-
-      ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:?Please set a value.}
+    image: "ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:-main}"
     pull_policy: "always"
     networks:
       - "spider-internal"
@@ -111,7 +108,7 @@ services:
   spider-worker:
     <<: *service_defaults
     image: >-
-      ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:?Please set a value.}
+      ${SPIDER_WORKER_CONTAINER_IMAGE_REF:-ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:-main}}
     pull_policy: "always"
     networks:
       - "default"

From bb36fec3b6dd5d97d9a6528d52c57aab0f4ff8eb Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Wed, 5 Aug 2026 23:05:01 -0400
Subject: [PATCH 04/21] Add clean task

---
 taskfiles/docker.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml
index d528a749e..f39d80f43 100644
--- a/taskfiles/docker.yaml
+++ b/taskfiles/docker.yaml
@@ -28,13 +28,13 @@ tasks:
     cmd: "docker compose --file compose.yaml config --quiet"
 
   compose:up:
-    vars:
-      WAIT_TIMEOUT: "{{.SPIDER_COMPOSE_WAIT_TIMEOUT | default \"120\"}}"
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"
-    cmd: >-
-      docker compose --file compose.yaml up --detach --wait
-      --wait-timeout "{{.WAIT_TIMEOUT}}"
+    cmd: "docker compose --file compose.yaml up --detach --wait"
 
   compose:down:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"
     cmd: "docker compose --file compose.yaml down"
+
+  compose:clean:
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: "docker compose --file compose.yaml down --volumes --remove-orphans"

From 07baeee340e6570f1554924a06758182486f17ef Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Thu, 6 Aug 2026 12:19:00 -0400
Subject: [PATCH 05/21] Fix config

---
 .gitignore                                   |  3 +++
 tools/deployment/spider-compose/.env.example | 20 ++++++++++++++++++++
 tools/deployment/spider-compose/compose.yaml | 17 +++++------------
 3 files changed, 28 insertions(+), 12 deletions(-)
 create mode 100644 tools/deployment/spider-compose/.env.example

diff --git a/.gitignore b/.gitignore
index 38add8926..717a7f6a1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,6 @@ build
 # IDE-related directories and files
 .idea
 cmake-build-debug
+
+# Local Docker Compose credentials
+tools/deployment/spider-compose/.env
diff --git a/tools/deployment/spider-compose/.env.example b/tools/deployment/spider-compose/.env.example
new file mode 100644
index 000000000..3a1b058b7
--- /dev/null
+++ b/tools/deployment/spider-compose/.env.example
@@ -0,0 +1,20 @@
+# Copy this file to `.env` and replace the development credentials before starting Spider.
+
+# Required database credentials.
+SPIDER_DATABASE_ROOT_PASSWORD=spider-root-password
+SPIDER_STORAGE_DB_PASSWORD=spider-password
+SPIDER_STORAGE_DB_USERNAME=spider-user
+
+# Optional deployment settings and their defaults.
+SPIDER_IMAGE_TAG=main
+SPIDER_PULL_POLICY=always
+SPIDER_RESTART_POLICY=on-failure:3
+SPIDER_SCHEDULER_LOG_LEVEL=INFO
+SPIDER_STORAGE_HOST_PORT=50051
+SPIDER_STORAGE_LOG_LEVEL=INFO
+SPIDER_STORAGE_PUBLISH_IP=127.0.0.1
+SPIDER_WORKER_LOG_LEVEL=INFO
+SPIDER_WORKER_REPLICAS=4
+
+# A complete worker image reference takes precedence over `SPIDER_IMAGE_TAG` when set.
+# SPIDER_WORKER_CONTAINER_IMAGE_REF=ghcr.io/y-scope/spider/worker:main
diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
index 6bffebdf1..b65d11fd3 100644
--- a/tools/deployment/spider-compose/compose.yaml
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -1,12 +1,9 @@
-# Required variables:
-# - SPIDER_STORAGE_DB_USERNAME
-# - SPIDER_STORAGE_DB_PASSWORD
-# - SPIDER_DATABASE_ROOT_PASSWORD
 name: "spider"
 
 x-service-defaults: &service_defaults
   logging:
     driver: "local"
+  pull_policy: "${SPIDER_PULL_POLICY:-always}"
   restart: "${SPIDER_RESTART_POLICY:-on-failure:3}"
   stop_grace_period: "60s"
 
@@ -21,7 +18,6 @@ services:
   spider-database:
     <<: *service_defaults
     image: "mariadb:10.11.16"
-    pull_policy: "always"
     networks:
       - "spider-internal"
     environment:
@@ -47,7 +43,6 @@ services:
   spider-storage:
     <<: *service_defaults
     image: "ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:-main}"
-    pull_policy: "always"
     networks:
       - "default"
       - "spider-internal"
@@ -56,11 +51,11 @@ services:
       "--config", "/etc/spider/storage.yaml",
     ]
     environment:
-      RUST_LOG: "${SPIDER_STORAGE_RUST_LOG:-INFO}"
+      RUST_LOG: "${SPIDER_STORAGE_LOG_LEVEL:-INFO}"
       SPIDER_STORAGE_DB_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?Please set a value.}"
       SPIDER_STORAGE_DB_USERNAME: "${SPIDER_STORAGE_DB_USERNAME:?Please set a value.}"
     ports:
-      - host_ip: "${SPIDER_STORAGE_HOST:-127.0.0.1}"
+      - host_ip: "${SPIDER_STORAGE_PUBLISH_IP:-127.0.0.1}"
         published: "${SPIDER_STORAGE_HOST_PORT:-50051}"
         target: 50051
     configs:
@@ -81,7 +76,6 @@ services:
   spider-scheduler:
     <<: *service_defaults
     image: "ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:-main}"
-    pull_policy: "always"
     networks:
       - "spider-internal"
     command: [
@@ -89,7 +83,7 @@ services:
       "--config", "/etc/spider/scheduler.yaml",
     ]
     environment:
-      RUST_LOG: "${SPIDER_SCHEDULER_RUST_LOG:-INFO}"
+      RUST_LOG: "${SPIDER_SCHEDULER_LOG_LEVEL:-INFO}"
     configs:
       - source: "spider-scheduler-config"
         target: "/etc/spider/scheduler.yaml"
@@ -109,7 +103,6 @@ services:
     <<: *service_defaults
     image: >-
       ${SPIDER_WORKER_CONTAINER_IMAGE_REF:-ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:-main}}
-    pull_policy: "always"
     networks:
       - "default"
       - "spider-internal"
@@ -118,7 +111,7 @@ services:
       "--config", "/etc/spider/worker.yaml",
     ]
     environment:
-      RUST_LOG: "${SPIDER_WORKER_RUST_LOG:-INFO}"
+      RUST_LOG: "${SPIDER_WORKER_LOG_LEVEL:-INFO}"
     configs:
       - source: "spider-worker-config"
         target: "/etc/spider/worker.yaml"

From ce11907cf5c0835640f00ea53b627c5da2bf9fd8 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Fri, 7 Aug 2026 00:37:22 -0400
Subject: [PATCH 06/21] Rename variables

---
 tools/deployment/spider-compose/.env.example | 2 +-
 tools/deployment/spider-compose/compose.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/deployment/spider-compose/.env.example b/tools/deployment/spider-compose/.env.example
index 3a1b058b7..dcaba6489 100644
--- a/tools/deployment/spider-compose/.env.example
+++ b/tools/deployment/spider-compose/.env.example
@@ -17,4 +17,4 @@ SPIDER_WORKER_LOG_LEVEL=INFO
 SPIDER_WORKER_REPLICAS=4
 
 # A complete worker image reference takes precedence over `SPIDER_IMAGE_TAG` when set.
-# SPIDER_WORKER_CONTAINER_IMAGE_REF=ghcr.io/y-scope/spider/worker:main
+# SPIDER_WORKER_IMAGE_REF=ghcr.io/y-scope/spider/worker:main
diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
index b65d11fd3..c8062bc9f 100644
--- a/tools/deployment/spider-compose/compose.yaml
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -102,7 +102,7 @@ services:
   spider-worker:
     <<: *service_defaults
     image: >-
-      ${SPIDER_WORKER_CONTAINER_IMAGE_REF:-ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:-main}}
+      ${SPIDER_WORKER_IMAGE_REF:-ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:-main}}
     networks:
       - "default"
       - "spider-internal"

From bc35203673dcefffcca8a51061101d5ef6844e66 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Tue, 11 Aug 2026 02:22:36 -0400
Subject: [PATCH 07/21] Expose user-facing config through env file

---
 taskfiles/docker.yaml                         |   2 +-
 tools/deployment/spider-compose/.env.example  |  52 +++++++--
 tools/deployment/spider-compose/compose.yaml  | 106 +++++++++++++-----
 .../spider-compose/config/scheduler.yaml      |  21 ----
 .../spider-compose/config/storage.yaml        |  12 --
 .../spider-compose/config/worker.yaml         |  16 ---
 6 files changed, 123 insertions(+), 86 deletions(-)
 delete mode 100644 tools/deployment/spider-compose/config/scheduler.yaml
 delete mode 100644 tools/deployment/spider-compose/config/storage.yaml
 delete mode 100644 tools/deployment/spider-compose/config/worker.yaml

diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml
index f39d80f43..9fc187acb 100644
--- a/taskfiles/docker.yaml
+++ b/taskfiles/docker.yaml
@@ -25,7 +25,7 @@ tasks:
 
   compose:validate:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"
-    cmd: "docker compose --file compose.yaml config --quiet"
+    cmd: "docker compose --env-file .env.example --file compose.yaml config --quiet"
 
   compose:up:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"
diff --git a/tools/deployment/spider-compose/.env.example b/tools/deployment/spider-compose/.env.example
index dcaba6489..e95e64352 100644
--- a/tools/deployment/spider-compose/.env.example
+++ b/tools/deployment/spider-compose/.env.example
@@ -1,20 +1,52 @@
 # Copy this file to `.env` and replace the development credentials before starting Spider.
 
-# Required database credentials.
+# Deployment settings.
+SPIDER_IMAGE_TAG=main
+SPIDER_PULL_POLICY=always
+SPIDER_RESTART_POLICY=on-failure:3
+
+# Database settings.
+SPIDER_DATABASE_IMAGE_REF=mariadb:10.11.16
+SPIDER_DATABASE_NAME=spider-db
+SPIDER_DATABASE_PORT=3306
 SPIDER_DATABASE_ROOT_PASSWORD=spider-root-password
 SPIDER_STORAGE_DB_PASSWORD=spider-password
 SPIDER_STORAGE_DB_USERNAME=spider-user
 
-# Optional deployment settings and their defaults.
-SPIDER_IMAGE_TAG=main
-SPIDER_PULL_POLICY=always
-SPIDER_RESTART_POLICY=on-failure:3
-SPIDER_SCHEDULER_LOG_LEVEL=INFO
-SPIDER_STORAGE_HOST_PORT=50051
+# Storage settings.
 SPIDER_STORAGE_LOG_LEVEL=INFO
-SPIDER_STORAGE_PUBLISH_IP=127.0.0.1
-SPIDER_WORKER_LOG_LEVEL=INFO
-SPIDER_WORKER_REPLICAS=4
+SPIDER_STORAGE_PORT=50051
+SPIDER_STORAGE_PUBLISHED_IP=127.0.0.1
+SPIDER_STORAGE_PUBLISHED_PORT=50051
+SPIDER_STORAGE_DB_MAX_CONNECTIONS=64
+SPIDER_STORAGE_READY_QUEUE_CLEANUP_CAPACITY=256
+SPIDER_STORAGE_READY_QUEUE_COMMIT_CAPACITY=256
+SPIDER_STORAGE_READY_QUEUE_TASK_CAPACITY=1048576
 
+# Scheduler settings.
+SPIDER_SCHEDULER_LOG_LEVEL=INFO
+SPIDER_SCHEDULER_PORT=50052
+SPIDER_SCHEDULER_CONNECTION_POOL_SIZE=4
+SPIDER_SCHEDULER_POLICY=round_robin
+SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY=16
+SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY=256
+SPIDER_SCHEDULER_COMMIT_READY_TASK_CAPACITY=256
+SPIDER_SCHEDULER_DISPATCH_QUEUE_CAPACITY=16
+SPIDER_SCHEDULER_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC=300
+SPIDER_SCHEDULER_READY_TASK_CAPACITY=1048576
+SPIDER_SCHEDULER_STORAGE_POLL_TIMEOUT_MS=10
+SPIDER_SCHEDULER_TICK_INTERVAL_MS=5
+
+# Worker settings.
 # A complete worker image reference takes precedence over `SPIDER_IMAGE_TAG` when set.
 # SPIDER_WORKER_IMAGE_REF=ghcr.io/y-scope/spider/worker:main
+SPIDER_WORKER_REPLICAS=4
+SPIDER_WORKER_LOG_LEVEL=INFO
+SPIDER_WORKER_CONNECTION_POOL_SIZE=4
+SPIDER_WORKER_SCHEDULER_HEARTBEAT_INTERVAL_SEC=1
+SPIDER_WORKER_SCHEDULER_POLL_WAIT_MS=1000
+SPIDER_WORKER_STORAGE_HEARTBEAT_INTERVAL_SEC=1
+SPIDER_WORKER_TASK_EXECUTOR_BIN_PATH=/usr/local/bin/spider-task-executor
+SPIDER_WORKER_INHERITED_ENV=[]
+SPIDER_WORKER_MAX_LOG_LINE_BYTES=65536
+SPIDER_WORKER_PACKAGE_DIR=/opt/spider/packages
diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
index c8062bc9f..02e3338b1 100644
--- a/tools/deployment/spider-compose/compose.yaml
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -3,8 +3,8 @@ name: "spider"
 x-service-defaults: &service_defaults
   logging:
     driver: "local"
-  pull_policy: "${SPIDER_PULL_POLICY:-always}"
-  restart: "${SPIDER_RESTART_POLICY:-on-failure:3}"
+  pull_policy: "${SPIDER_PULL_POLICY:?}"
+  restart: "${SPIDER_RESTART_POLICY:?}"
   stop_grace_period: "60s"
 
 x-healthcheck-defaults: &healthcheck_defaults
@@ -17,14 +17,17 @@ x-healthcheck-defaults: &healthcheck_defaults
 services:
   spider-database:
     <<: *service_defaults
-    image: "mariadb:10.11.16"
+    image: "${SPIDER_DATABASE_IMAGE_REF:?}"
     networks:
       - "spider-internal"
+    command: [
+      "--port=${SPIDER_DATABASE_PORT:?}",
+    ]
     environment:
-      MYSQL_DATABASE: "spider-db"
-      MYSQL_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?Please set a value.}"
-      MYSQL_ROOT_PASSWORD: "${SPIDER_DATABASE_ROOT_PASSWORD:?Please set a value.}"
-      MYSQL_USER: "${SPIDER_STORAGE_DB_USERNAME:?Please set a value.}"
+      MYSQL_DATABASE: "${SPIDER_DATABASE_NAME:?}"
+      MYSQL_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?}"
+      MYSQL_ROOT_PASSWORD: "${SPIDER_DATABASE_ROOT_PASSWORD:?}"
+      MYSQL_USER: "${SPIDER_STORAGE_DB_USERNAME:?}"
     volumes:
       - type: "volume"
         source: "spider-database-data"
@@ -36,13 +39,14 @@ services:
         "mysqladmin", "ping",
         "--silent",
         "-h", "127.0.0.1",
-        "-u", "${SPIDER_STORAGE_DB_USERNAME:?Please set a value.}",
-        "--password=${SPIDER_STORAGE_DB_PASSWORD:?Please set a value.}",
+        "-P", "${SPIDER_DATABASE_PORT:?}",
+        "-u", "${SPIDER_STORAGE_DB_USERNAME:?}",
+        "--password=${SPIDER_STORAGE_DB_PASSWORD:?}",
       ]
 
   spider-storage:
     <<: *service_defaults
-    image: "ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:-main}"
+    image: "ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:?}"
     networks:
       - "default"
       - "spider-internal"
@@ -51,13 +55,13 @@ services:
       "--config", "/etc/spider/storage.yaml",
     ]
     environment:
-      RUST_LOG: "${SPIDER_STORAGE_LOG_LEVEL:-INFO}"
-      SPIDER_STORAGE_DB_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?Please set a value.}"
-      SPIDER_STORAGE_DB_USERNAME: "${SPIDER_STORAGE_DB_USERNAME:?Please set a value.}"
+      RUST_LOG: "${SPIDER_STORAGE_LOG_LEVEL:?}"
+      SPIDER_STORAGE_DB_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?}"
+      SPIDER_STORAGE_DB_USERNAME: "${SPIDER_STORAGE_DB_USERNAME:?}"
     ports:
-      - host_ip: "${SPIDER_STORAGE_PUBLISH_IP:-127.0.0.1}"
-        published: "${SPIDER_STORAGE_HOST_PORT:-50051}"
-        target: 50051
+      - host_ip: "${SPIDER_STORAGE_PUBLISHED_IP:?}"
+        published: "${SPIDER_STORAGE_PUBLISHED_PORT:?}"
+        target: "${SPIDER_STORAGE_PORT:?}"
     configs:
       - source: "spider-storage-config"
         target: "/etc/spider/storage.yaml"
@@ -70,12 +74,12 @@ services:
       test: [
         "CMD",
         "timeout", "2",
-        "bash", "-c", "</dev/tcp/127.0.0.1/50051",
+        "bash", "-c", "</dev/tcp/127.0.0.1/${SPIDER_STORAGE_PORT:?}",
       ]
 
   spider-scheduler:
     <<: *service_defaults
-    image: "ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:-main}"
+    image: "ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:?}"
     networks:
       - "spider-internal"
     command: [
@@ -83,7 +87,7 @@ services:
       "--config", "/etc/spider/scheduler.yaml",
     ]
     environment:
-      RUST_LOG: "${SPIDER_SCHEDULER_LOG_LEVEL:-INFO}"
+      RUST_LOG: "${SPIDER_SCHEDULER_LOG_LEVEL:?}"
     configs:
       - source: "spider-scheduler-config"
         target: "/etc/spider/scheduler.yaml"
@@ -96,13 +100,13 @@ services:
       test: [
         "CMD",
         "timeout", "2",
-        "bash", "-c", "</dev/tcp/127.0.0.1/50052",
+        "bash", "-c", "</dev/tcp/127.0.0.1/${SPIDER_SCHEDULER_PORT:?}",
       ]
 
   spider-worker:
     <<: *service_defaults
     image: >-
-      ${SPIDER_WORKER_IMAGE_REF:-ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:-main}}
+      ${SPIDER_WORKER_IMAGE_REF:-ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:?}}
     networks:
       - "default"
       - "spider-internal"
@@ -111,7 +115,7 @@ services:
       "--config", "/etc/spider/worker.yaml",
     ]
     environment:
-      RUST_LOG: "${SPIDER_WORKER_LOG_LEVEL:-INFO}"
+      RUST_LOG: "${SPIDER_WORKER_LOG_LEVEL:?}"
     configs:
       - source: "spider-worker-config"
         target: "/etc/spider/worker.yaml"
@@ -123,15 +127,65 @@ services:
         condition: "service_healthy"
         restart: true
     deploy:
-      replicas: "${SPIDER_WORKER_REPLICAS:-4}"
+      replicas: "${SPIDER_WORKER_REPLICAS:?}"
 
 configs:
   spider-scheduler-config:
-    file: "./config/scheduler.yaml"
+    content: |
+      connection_pool_size: ${SPIDER_SCHEDULER_CONNECTION_POOL_SIZE:?}
+      host: "0.0.0.0"
+      port: ${SPIDER_SCHEDULER_PORT:?}
+      runtime:
+        advertised_endpoint:
+          host: "spider-scheduler"
+          port: ${SPIDER_SCHEDULER_PORT:?}
+        scheduler:
+          policy: "${SPIDER_SCHEDULER_POLICY:?}"
+          config:
+            active_job_queue_capacity: ${SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY:?}
+            cleanup_ready_task_capacity: ${SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY:?}
+            commit_ready_task_capacity: ${SPIDER_SCHEDULER_COMMIT_READY_TASK_CAPACITY:?}
+            dispatch_queue_capacity: ${SPIDER_SCHEDULER_DISPATCH_QUEUE_CAPACITY:?}
+            finalizing_job_expiration_timeout_sec:
+              ${SPIDER_SCHEDULER_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC:?}
+            ready_task_capacity: ${SPIDER_SCHEDULER_READY_TASK_CAPACITY:?}
+            storage_poll_timeout_ms: ${SPIDER_SCHEDULER_STORAGE_POLL_TIMEOUT_MS:?}
+            tick_interval_ms: ${SPIDER_SCHEDULER_TICK_INTERVAL_MS:?}
+      storage_endpoint:
+        host: "spider-storage"
+        port: ${SPIDER_STORAGE_PORT:?}
   spider-storage-config:
-    file: "./config/storage.yaml"
+    content: |
+      host: "0.0.0.0"
+      port: ${SPIDER_STORAGE_PORT:?}
+      runtime:
+        db_config:
+          host: "spider-database"
+          max_connections: ${SPIDER_STORAGE_DB_MAX_CONNECTIONS:?}
+          name: "${SPIDER_DATABASE_NAME:?}"
+          port: ${SPIDER_DATABASE_PORT:?}
+        ready_queue_config:
+          cleanup_capacity: ${SPIDER_STORAGE_READY_QUEUE_CLEANUP_CAPACITY:?}
+          commit_capacity: ${SPIDER_STORAGE_READY_QUEUE_COMMIT_CAPACITY:?}
+          task_capacity: ${SPIDER_STORAGE_READY_QUEUE_TASK_CAPACITY:?}
   spider-worker-config:
-    file: "./config/worker.yaml"
+    content: |
+      connection_pool_size: ${SPIDER_WORKER_CONNECTION_POOL_SIZE:?}
+      liveness:
+        scheduler_heartbeat_interval_sec: ${SPIDER_WORKER_SCHEDULER_HEARTBEAT_INTERVAL_SEC:?}
+        storage_heartbeat_interval_sec: ${SPIDER_WORKER_STORAGE_HEARTBEAT_INTERVAL_SEC:?}
+      scheduler:
+        host: "spider-scheduler"
+        port: ${SPIDER_SCHEDULER_PORT:?}
+      scheduler_poll_wait_ms: ${SPIDER_WORKER_SCHEDULER_POLL_WAIT_MS:?}
+      storage:
+        host: "spider-storage"
+        port: ${SPIDER_STORAGE_PORT:?}
+      task_executor:
+        bin_path: "${SPIDER_WORKER_TASK_EXECUTOR_BIN_PATH:?}"
+        inherited_env: ${SPIDER_WORKER_INHERITED_ENV:?}
+        max_log_line_bytes: ${SPIDER_WORKER_MAX_LOG_LINE_BYTES:?}
+        package_dir: "${SPIDER_WORKER_PACKAGE_DIR:?}"
 
 networks:
   default:
diff --git a/tools/deployment/spider-compose/config/scheduler.yaml b/tools/deployment/spider-compose/config/scheduler.yaml
deleted file mode 100644
index 6989a0e19..000000000
--- a/tools/deployment/spider-compose/config/scheduler.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-connection_pool_size: 4
-host: "0.0.0.0"
-port: 50052
-runtime:
-  advertised_endpoint:
-    host: "spider-scheduler"
-    port: 50052
-  scheduler:
-    policy: "round_robin"
-    config:
-      active_job_queue_capacity: 64
-      cleanup_ready_task_capacity: 256
-      commit_ready_task_capacity: 256
-      dispatch_queue_capacity: 64
-      finalizing_job_expiration_timeout_sec: 300
-      ready_task_capacity: 65536
-      storage_poll_timeout_ms: 10
-      tick_interval_ms: 5
-storage_endpoint:
-  host: "spider-storage"
-  port: 50051
diff --git a/tools/deployment/spider-compose/config/storage.yaml b/tools/deployment/spider-compose/config/storage.yaml
deleted file mode 100644
index 70fef631b..000000000
--- a/tools/deployment/spider-compose/config/storage.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-host: "0.0.0.0"
-port: 50051
-runtime:
-  db_config:
-    host: "spider-database"
-    max_connections: 64
-    name: "spider-db"
-    port: 3306
-  ready_queue_config:
-    cleanup_capacity: 256
-    commit_capacity: 256
-    task_capacity: 1048576
diff --git a/tools/deployment/spider-compose/config/worker.yaml b/tools/deployment/spider-compose/config/worker.yaml
deleted file mode 100644
index 79faa5916..000000000
--- a/tools/deployment/spider-compose/config/worker.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-connection_pool_size: 4
-liveness:
-  scheduler_heartbeat_interval_sec: 10
-  storage_heartbeat_interval_sec: 10
-scheduler:
-  host: "spider-scheduler"
-  port: 50052
-scheduler_poll_wait_ms: 1000
-storage:
-  host: "spider-storage"
-  port: 50051
-task_executor:
-  bin_path: "/usr/local/bin/spider-task-executor"
-  inherited_env: []
-  max_log_line_bytes: 65536
-  package_dir: "/opt/spider/packages"

From d3dbf06dc19ac72c7855a0c803bfab5327499cb0 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Tue, 11 Aug 2026 02:43:17 -0400
Subject: [PATCH 08/21] Fix generated storage config keys

---
 tools/deployment/spider-compose/compose.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
index 02e3338b1..25cf24fdf 100644
--- a/tools/deployment/spider-compose/compose.yaml
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -159,12 +159,12 @@ configs:
       host: "0.0.0.0"
       port: ${SPIDER_STORAGE_PORT:?}
       runtime:
-        db_config:
+        db:
           host: "spider-database"
           max_connections: ${SPIDER_STORAGE_DB_MAX_CONNECTIONS:?}
           name: "${SPIDER_DATABASE_NAME:?}"
           port: ${SPIDER_DATABASE_PORT:?}
-        ready_queue_config:
+        ready_queue:
           cleanup_capacity: ${SPIDER_STORAGE_READY_QUEUE_CLEANUP_CAPACITY:?}
           commit_capacity: ${SPIDER_STORAGE_READY_QUEUE_COMMIT_CAPACITY:?}
           task_capacity: ${SPIDER_STORAGE_READY_QUEUE_TASK_CAPACITY:?}

From b3eed390ce09f86b3bdaec00b5b478d7837225b8 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Tue, 11 Aug 2026 18:00:04 -0400
Subject: [PATCH 09/21] Add defaults for Compose configuration

---
 tools/deployment/spider-compose/.env.example |   4 +-
 tools/deployment/spider-compose/compose.yaml | 110 +++++++++----------
 2 files changed, 57 insertions(+), 57 deletions(-)

diff --git a/tools/deployment/spider-compose/.env.example b/tools/deployment/spider-compose/.env.example
index e95e64352..d467e4434 100644
--- a/tools/deployment/spider-compose/.env.example
+++ b/tools/deployment/spider-compose/.env.example
@@ -1,4 +1,4 @@
-# Copy this file to `.env` and replace the development credentials before starting Spider.
+# Optionally copy this file to `.env` to override Spider's embedded defaults.
 
 # Deployment settings.
 SPIDER_IMAGE_TAG=main
@@ -26,7 +26,7 @@ SPIDER_STORAGE_READY_QUEUE_TASK_CAPACITY=1048576
 # Scheduler settings.
 SPIDER_SCHEDULER_LOG_LEVEL=INFO
 SPIDER_SCHEDULER_PORT=50052
-SPIDER_SCHEDULER_CONNECTION_POOL_SIZE=4
+SPIDER_SCHEDULER_CONNECTION_POOL_SIZE=8
 SPIDER_SCHEDULER_POLICY=round_robin
 SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY=16
 SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY=256
diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
index 25cf24fdf..c41642f21 100644
--- a/tools/deployment/spider-compose/compose.yaml
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -3,8 +3,8 @@ name: "spider"
 x-service-defaults: &service_defaults
   logging:
     driver: "local"
-  pull_policy: "${SPIDER_PULL_POLICY:?}"
-  restart: "${SPIDER_RESTART_POLICY:?}"
+  pull_policy: "${SPIDER_PULL_POLICY:-always}"
+  restart: "${SPIDER_RESTART_POLICY:-on-failure:3}"
   stop_grace_period: "60s"
 
 x-healthcheck-defaults: &healthcheck_defaults
@@ -17,17 +17,17 @@ x-healthcheck-defaults: &healthcheck_defaults
 services:
   spider-database:
     <<: *service_defaults
-    image: "${SPIDER_DATABASE_IMAGE_REF:?}"
+    image: "${SPIDER_DATABASE_IMAGE_REF:-mariadb:10.11.16}"
     networks:
       - "spider-internal"
     command: [
-      "--port=${SPIDER_DATABASE_PORT:?}",
+      "--port=${SPIDER_DATABASE_PORT:-3306}",
     ]
     environment:
-      MYSQL_DATABASE: "${SPIDER_DATABASE_NAME:?}"
-      MYSQL_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?}"
-      MYSQL_ROOT_PASSWORD: "${SPIDER_DATABASE_ROOT_PASSWORD:?}"
-      MYSQL_USER: "${SPIDER_STORAGE_DB_USERNAME:?}"
+      MYSQL_DATABASE: "${SPIDER_DATABASE_NAME:-spider-db}"
+      MYSQL_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:-spider-password}"
+      MYSQL_ROOT_PASSWORD: "${SPIDER_DATABASE_ROOT_PASSWORD:-spider-root-password}"
+      MYSQL_USER: "${SPIDER_STORAGE_DB_USERNAME:-spider-user}"
     volumes:
       - type: "volume"
         source: "spider-database-data"
@@ -39,14 +39,14 @@ services:
         "mysqladmin", "ping",
         "--silent",
         "-h", "127.0.0.1",
-        "-P", "${SPIDER_DATABASE_PORT:?}",
-        "-u", "${SPIDER_STORAGE_DB_USERNAME:?}",
-        "--password=${SPIDER_STORAGE_DB_PASSWORD:?}",
+        "-P", "${SPIDER_DATABASE_PORT:-3306}",
+        "-u", "${SPIDER_STORAGE_DB_USERNAME:-spider-user}",
+        "--password=${SPIDER_STORAGE_DB_PASSWORD:-spider-password}",
       ]
 
   spider-storage:
     <<: *service_defaults
-    image: "ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:?}"
+    image: "ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:-main}"
     networks:
       - "default"
       - "spider-internal"
@@ -55,13 +55,13 @@ services:
       "--config", "/etc/spider/storage.yaml",
     ]
     environment:
-      RUST_LOG: "${SPIDER_STORAGE_LOG_LEVEL:?}"
-      SPIDER_STORAGE_DB_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:?}"
-      SPIDER_STORAGE_DB_USERNAME: "${SPIDER_STORAGE_DB_USERNAME:?}"
+      RUST_LOG: "${SPIDER_STORAGE_LOG_LEVEL:-INFO}"
+      SPIDER_STORAGE_DB_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:-spider-password}"
+      SPIDER_STORAGE_DB_USERNAME: "${SPIDER_STORAGE_DB_USERNAME:-spider-user}"
     ports:
-      - host_ip: "${SPIDER_STORAGE_PUBLISHED_IP:?}"
-        published: "${SPIDER_STORAGE_PUBLISHED_PORT:?}"
-        target: "${SPIDER_STORAGE_PORT:?}"
+      - host_ip: "${SPIDER_STORAGE_PUBLISHED_IP:-127.0.0.1}"
+        published: "${SPIDER_STORAGE_PUBLISHED_PORT:-50051}"
+        target: "${SPIDER_STORAGE_PORT:-50051}"
     configs:
       - source: "spider-storage-config"
         target: "/etc/spider/storage.yaml"
@@ -74,12 +74,12 @@ services:
       test: [
         "CMD",
         "timeout", "2",
-        "bash", "-c", "</dev/tcp/127.0.0.1/${SPIDER_STORAGE_PORT:?}",
+        "bash", "-c", "</dev/tcp/127.0.0.1/${SPIDER_STORAGE_PORT:-50051}",
       ]
 
   spider-scheduler:
     <<: *service_defaults
-    image: "ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:?}"
+    image: "ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:-main}"
     networks:
       - "spider-internal"
     command: [
@@ -87,7 +87,7 @@ services:
       "--config", "/etc/spider/scheduler.yaml",
     ]
     environment:
-      RUST_LOG: "${SPIDER_SCHEDULER_LOG_LEVEL:?}"
+      RUST_LOG: "${SPIDER_SCHEDULER_LOG_LEVEL:-INFO}"
     configs:
       - source: "spider-scheduler-config"
         target: "/etc/spider/scheduler.yaml"
@@ -100,13 +100,13 @@ services:
       test: [
         "CMD",
         "timeout", "2",
-        "bash", "-c", "</dev/tcp/127.0.0.1/${SPIDER_SCHEDULER_PORT:?}",
+        "bash", "-c", "</dev/tcp/127.0.0.1/${SPIDER_SCHEDULER_PORT:-50052}",
       ]
 
   spider-worker:
     <<: *service_defaults
     image: >-
-      ${SPIDER_WORKER_IMAGE_REF:-ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:?}}
+      ${SPIDER_WORKER_IMAGE_REF:-ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:-main}}
     networks:
       - "default"
       - "spider-internal"
@@ -115,7 +115,7 @@ services:
       "--config", "/etc/spider/worker.yaml",
     ]
     environment:
-      RUST_LOG: "${SPIDER_WORKER_LOG_LEVEL:?}"
+      RUST_LOG: "${SPIDER_WORKER_LOG_LEVEL:-INFO}"
     configs:
       - source: "spider-worker-config"
         target: "/etc/spider/worker.yaml"
@@ -127,65 +127,65 @@ services:
         condition: "service_healthy"
         restart: true
     deploy:
-      replicas: "${SPIDER_WORKER_REPLICAS:?}"
+      replicas: "${SPIDER_WORKER_REPLICAS:-4}"
 
 configs:
   spider-scheduler-config:
     content: |
-      connection_pool_size: ${SPIDER_SCHEDULER_CONNECTION_POOL_SIZE:?}
+      connection_pool_size: ${SPIDER_SCHEDULER_CONNECTION_POOL_SIZE:-8}
       host: "0.0.0.0"
-      port: ${SPIDER_SCHEDULER_PORT:?}
+      port: ${SPIDER_SCHEDULER_PORT:-50052}
       runtime:
         advertised_endpoint:
           host: "spider-scheduler"
-          port: ${SPIDER_SCHEDULER_PORT:?}
+          port: ${SPIDER_SCHEDULER_PORT:-50052}
         scheduler:
-          policy: "${SPIDER_SCHEDULER_POLICY:?}"
+          policy: "${SPIDER_SCHEDULER_POLICY:-round_robin}"
           config:
-            active_job_queue_capacity: ${SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY:?}
-            cleanup_ready_task_capacity: ${SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY:?}
-            commit_ready_task_capacity: ${SPIDER_SCHEDULER_COMMIT_READY_TASK_CAPACITY:?}
-            dispatch_queue_capacity: ${SPIDER_SCHEDULER_DISPATCH_QUEUE_CAPACITY:?}
+            active_job_queue_capacity: ${SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY:-16}
+            cleanup_ready_task_capacity: ${SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY:-256}
+            commit_ready_task_capacity: ${SPIDER_SCHEDULER_COMMIT_READY_TASK_CAPACITY:-256}
+            dispatch_queue_capacity: ${SPIDER_SCHEDULER_DISPATCH_QUEUE_CAPACITY:-16}
             finalizing_job_expiration_timeout_sec:
-              ${SPIDER_SCHEDULER_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC:?}
-            ready_task_capacity: ${SPIDER_SCHEDULER_READY_TASK_CAPACITY:?}
-            storage_poll_timeout_ms: ${SPIDER_SCHEDULER_STORAGE_POLL_TIMEOUT_MS:?}
-            tick_interval_ms: ${SPIDER_SCHEDULER_TICK_INTERVAL_MS:?}
+              ${SPIDER_SCHEDULER_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC:-300}
+            ready_task_capacity: ${SPIDER_SCHEDULER_READY_TASK_CAPACITY:-1048576}
+            storage_poll_timeout_ms: ${SPIDER_SCHEDULER_STORAGE_POLL_TIMEOUT_MS:-10}
+            tick_interval_ms: ${SPIDER_SCHEDULER_TICK_INTERVAL_MS:-5}
       storage_endpoint:
         host: "spider-storage"
-        port: ${SPIDER_STORAGE_PORT:?}
+        port: ${SPIDER_STORAGE_PORT:-50051}
   spider-storage-config:
     content: |
       host: "0.0.0.0"
-      port: ${SPIDER_STORAGE_PORT:?}
+      port: ${SPIDER_STORAGE_PORT:-50051}
       runtime:
         db:
           host: "spider-database"
-          max_connections: ${SPIDER_STORAGE_DB_MAX_CONNECTIONS:?}
-          name: "${SPIDER_DATABASE_NAME:?}"
-          port: ${SPIDER_DATABASE_PORT:?}
+          max_connections: ${SPIDER_STORAGE_DB_MAX_CONNECTIONS:-64}
+          name: "${SPIDER_DATABASE_NAME:-spider-db}"
+          port: ${SPIDER_DATABASE_PORT:-3306}
         ready_queue:
-          cleanup_capacity: ${SPIDER_STORAGE_READY_QUEUE_CLEANUP_CAPACITY:?}
-          commit_capacity: ${SPIDER_STORAGE_READY_QUEUE_COMMIT_CAPACITY:?}
-          task_capacity: ${SPIDER_STORAGE_READY_QUEUE_TASK_CAPACITY:?}
+          cleanup_capacity: ${SPIDER_STORAGE_READY_QUEUE_CLEANUP_CAPACITY:-256}
+          commit_capacity: ${SPIDER_STORAGE_READY_QUEUE_COMMIT_CAPACITY:-256}
+          task_capacity: ${SPIDER_STORAGE_READY_QUEUE_TASK_CAPACITY:-1048576}
   spider-worker-config:
     content: |
-      connection_pool_size: ${SPIDER_WORKER_CONNECTION_POOL_SIZE:?}
+      connection_pool_size: ${SPIDER_WORKER_CONNECTION_POOL_SIZE:-4}
       liveness:
-        scheduler_heartbeat_interval_sec: ${SPIDER_WORKER_SCHEDULER_HEARTBEAT_INTERVAL_SEC:?}
-        storage_heartbeat_interval_sec: ${SPIDER_WORKER_STORAGE_HEARTBEAT_INTERVAL_SEC:?}
+        scheduler_heartbeat_interval_sec: ${SPIDER_WORKER_SCHEDULER_HEARTBEAT_INTERVAL_SEC:-1}
+        storage_heartbeat_interval_sec: ${SPIDER_WORKER_STORAGE_HEARTBEAT_INTERVAL_SEC:-1}
       scheduler:
         host: "spider-scheduler"
-        port: ${SPIDER_SCHEDULER_PORT:?}
-      scheduler_poll_wait_ms: ${SPIDER_WORKER_SCHEDULER_POLL_WAIT_MS:?}
+        port: ${SPIDER_SCHEDULER_PORT:-50052}
+      scheduler_poll_wait_ms: ${SPIDER_WORKER_SCHEDULER_POLL_WAIT_MS:-1000}
       storage:
         host: "spider-storage"
-        port: ${SPIDER_STORAGE_PORT:?}
+        port: ${SPIDER_STORAGE_PORT:-50051}
       task_executor:
-        bin_path: "${SPIDER_WORKER_TASK_EXECUTOR_BIN_PATH:?}"
-        inherited_env: ${SPIDER_WORKER_INHERITED_ENV:?}
-        max_log_line_bytes: ${SPIDER_WORKER_MAX_LOG_LINE_BYTES:?}
-        package_dir: "${SPIDER_WORKER_PACKAGE_DIR:?}"
+        bin_path: "${SPIDER_WORKER_TASK_EXECUTOR_BIN_PATH:-/usr/local/bin/spider-task-executor}"
+        inherited_env: ${SPIDER_WORKER_INHERITED_ENV:-[]}
+        max_log_line_bytes: ${SPIDER_WORKER_MAX_LOG_LINE_BYTES:-65536}
+        package_dir: "${SPIDER_WORKER_PACKAGE_DIR:-/opt/spider/packages}"
 
 networks:
   default:

From 07f764e0887b92757a3ec5575fc8920793cea747 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Tue, 11 Aug 2026 21:07:24 -0400
Subject: [PATCH 10/21] Update Compose runtime configuration

---
 tools/deployment/spider-compose/.env.example | 14 +++++++++---
 tools/deployment/spider-compose/compose.yaml | 23 ++++++++++++++++----
 2 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/tools/deployment/spider-compose/.env.example b/tools/deployment/spider-compose/.env.example
index d467e4434..8b450765b 100644
--- a/tools/deployment/spider-compose/.env.example
+++ b/tools/deployment/spider-compose/.env.example
@@ -19,14 +19,22 @@ SPIDER_STORAGE_PORT=50051
 SPIDER_STORAGE_PUBLISHED_IP=127.0.0.1
 SPIDER_STORAGE_PUBLISHED_PORT=50051
 SPIDER_STORAGE_DB_MAX_CONNECTIONS=64
-SPIDER_STORAGE_READY_QUEUE_CLEANUP_CAPACITY=256
-SPIDER_STORAGE_READY_QUEUE_COMMIT_CAPACITY=256
-SPIDER_STORAGE_READY_QUEUE_TASK_CAPACITY=1048576
+SPIDER_STORAGE_INBOUND_QUEUE_CLEANUP_CAPACITY=256
+SPIDER_STORAGE_INBOUND_QUEUE_COMMIT_CAPACITY=256
+SPIDER_STORAGE_INBOUND_QUEUE_TASK_CAPACITY=1048576
+SPIDER_STORAGE_JOB_CACHE_GC_INTERVAL_SEC=30
+SPIDER_STORAGE_JOB_CACHE_TERMINATED_JOB_RETENTION_SEC=300
+SPIDER_STORAGE_TASK_INSTANCE_POOL_EXECUTION_MANAGER_STALE_CUTOFF_SEC=60
+SPIDER_STORAGE_TASK_INSTANCE_POOL_GC_INTERVAL_SEC=30
+SPIDER_STORAGE_TASK_INSTANCE_POOL_MESSAGE_CHANNEL_CAPACITY=128
 
 # Scheduler settings.
 SPIDER_SCHEDULER_LOG_LEVEL=INFO
 SPIDER_SCHEDULER_PORT=50052
 SPIDER_SCHEDULER_CONNECTION_POOL_SIZE=8
+SPIDER_SCHEDULER_STOP_TIMEOUT_SEC=30
+SPIDER_SCHEDULER_EM_REGISTRY_DEAD_EM_CUTOFF_SEC=30
+SPIDER_SCHEDULER_EM_REGISTRY_LIVENESS_TRACKING_INTERVAL_MS=1000
 SPIDER_SCHEDULER_POLICY=round_robin
 SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY=16
 SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY=256
diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
index c41642f21..decbb4f75 100644
--- a/tools/deployment/spider-compose/compose.yaml
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -139,6 +139,10 @@ configs:
         advertised_endpoint:
           host: "spider-scheduler"
           port: ${SPIDER_SCHEDULER_PORT:-50052}
+        em_registry:
+          dead_em_cutoff_sec: ${SPIDER_SCHEDULER_EM_REGISTRY_DEAD_EM_CUTOFF_SEC:-30}
+          liveness_tracking_interval_ms:
+            ${SPIDER_SCHEDULER_EM_REGISTRY_LIVENESS_TRACKING_INTERVAL_MS:-1000}
         scheduler:
           policy: "${SPIDER_SCHEDULER_POLICY:-round_robin}"
           config:
@@ -151,6 +155,7 @@ configs:
             ready_task_capacity: ${SPIDER_SCHEDULER_READY_TASK_CAPACITY:-1048576}
             storage_poll_timeout_ms: ${SPIDER_SCHEDULER_STORAGE_POLL_TIMEOUT_MS:-10}
             tick_interval_ms: ${SPIDER_SCHEDULER_TICK_INTERVAL_MS:-5}
+        stop_timeout_sec: ${SPIDER_SCHEDULER_STOP_TIMEOUT_SEC:-30}
       storage_endpoint:
         host: "spider-storage"
         port: ${SPIDER_STORAGE_PORT:-50051}
@@ -164,10 +169,20 @@ configs:
           max_connections: ${SPIDER_STORAGE_DB_MAX_CONNECTIONS:-64}
           name: "${SPIDER_DATABASE_NAME:-spider-db}"
           port: ${SPIDER_DATABASE_PORT:-3306}
-        ready_queue:
-          cleanup_capacity: ${SPIDER_STORAGE_READY_QUEUE_CLEANUP_CAPACITY:-256}
-          commit_capacity: ${SPIDER_STORAGE_READY_QUEUE_COMMIT_CAPACITY:-256}
-          task_capacity: ${SPIDER_STORAGE_READY_QUEUE_TASK_CAPACITY:-1048576}
+        inbound_queue:
+          cleanup_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_CLEANUP_CAPACITY:-256}
+          commit_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_COMMIT_CAPACITY:-256}
+          task_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_TASK_CAPACITY:-1048576}
+        job_cache_gc:
+          gc_interval_sec: ${SPIDER_STORAGE_JOB_CACHE_GC_INTERVAL_SEC:-30}
+          terminated_job_retention_sec:
+            ${SPIDER_STORAGE_JOB_CACHE_TERMINATED_JOB_RETENTION_SEC:-300}
+        task_instance_pool:
+          execution_manager_stale_cutoff_sec:
+            ${SPIDER_STORAGE_TASK_INSTANCE_POOL_EXECUTION_MANAGER_STALE_CUTOFF_SEC:-60}
+          gc_interval_sec: ${SPIDER_STORAGE_TASK_INSTANCE_POOL_GC_INTERVAL_SEC:-30}
+          message_channel_capacity:
+            ${SPIDER_STORAGE_TASK_INSTANCE_POOL_MESSAGE_CHANNEL_CAPACITY:-128}
   spider-worker-config:
     content: |
       connection_pool_size: ${SPIDER_WORKER_CONNECTION_POOL_SIZE:-4}

From ab5a157fe9b361a97cecc4be8a7cf6b335f31262 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Wed, 12 Aug 2026 11:54:48 -0400
Subject: [PATCH 11/21] Split configs out

---
 tools/deployment/spider-compose/compose.yaml  | 78 ++-----------------
 .../spider-compose/configs/scheduler.yaml     | 30 +++++++
 .../spider-compose/configs/storage.yaml       | 25 ++++++
 .../spider-compose/configs/worker.yaml        | 19 +++++
 4 files changed, 79 insertions(+), 73 deletions(-)
 create mode 100644 tools/deployment/spider-compose/configs/scheduler.yaml
 create mode 100644 tools/deployment/spider-compose/configs/storage.yaml
 create mode 100644 tools/deployment/spider-compose/configs/worker.yaml

diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml
index decbb4f75..6dcb3e676 100644
--- a/tools/deployment/spider-compose/compose.yaml
+++ b/tools/deployment/spider-compose/compose.yaml
@@ -1,5 +1,10 @@
 name: "spider"
 
+include:
+  - "./configs/scheduler.yaml"
+  - "./configs/storage.yaml"
+  - "./configs/worker.yaml"
+
 x-service-defaults: &service_defaults
   logging:
     driver: "local"
@@ -129,79 +134,6 @@ services:
     deploy:
       replicas: "${SPIDER_WORKER_REPLICAS:-4}"
 
-configs:
-  spider-scheduler-config:
-    content: |
-      connection_pool_size: ${SPIDER_SCHEDULER_CONNECTION_POOL_SIZE:-8}
-      host: "0.0.0.0"
-      port: ${SPIDER_SCHEDULER_PORT:-50052}
-      runtime:
-        advertised_endpoint:
-          host: "spider-scheduler"
-          port: ${SPIDER_SCHEDULER_PORT:-50052}
-        em_registry:
-          dead_em_cutoff_sec: ${SPIDER_SCHEDULER_EM_REGISTRY_DEAD_EM_CUTOFF_SEC:-30}
-          liveness_tracking_interval_ms:
-            ${SPIDER_SCHEDULER_EM_REGISTRY_LIVENESS_TRACKING_INTERVAL_MS:-1000}
-        scheduler:
-          policy: "${SPIDER_SCHEDULER_POLICY:-round_robin}"
-          config:
-            active_job_queue_capacity: ${SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY:-16}
-            cleanup_ready_task_capacity: ${SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY:-256}
-            commit_ready_task_capacity: ${SPIDER_SCHEDULER_COMMIT_READY_TASK_CAPACITY:-256}
-            dispatch_queue_capacity: ${SPIDER_SCHEDULER_DISPATCH_QUEUE_CAPACITY:-16}
-            finalizing_job_expiration_timeout_sec:
-              ${SPIDER_SCHEDULER_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC:-300}
-            ready_task_capacity: ${SPIDER_SCHEDULER_READY_TASK_CAPACITY:-1048576}
-            storage_poll_timeout_ms: ${SPIDER_SCHEDULER_STORAGE_POLL_TIMEOUT_MS:-10}
-            tick_interval_ms: ${SPIDER_SCHEDULER_TICK_INTERVAL_MS:-5}
-        stop_timeout_sec: ${SPIDER_SCHEDULER_STOP_TIMEOUT_SEC:-30}
-      storage_endpoint:
-        host: "spider-storage"
-        port: ${SPIDER_STORAGE_PORT:-50051}
-  spider-storage-config:
-    content: |
-      host: "0.0.0.0"
-      port: ${SPIDER_STORAGE_PORT:-50051}
-      runtime:
-        db:
-          host: "spider-database"
-          max_connections: ${SPIDER_STORAGE_DB_MAX_CONNECTIONS:-64}
-          name: "${SPIDER_DATABASE_NAME:-spider-db}"
-          port: ${SPIDER_DATABASE_PORT:-3306}
-        inbound_queue:
-          cleanup_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_CLEANUP_CAPACITY:-256}
-          commit_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_COMMIT_CAPACITY:-256}
-          task_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_TASK_CAPACITY:-1048576}
-        job_cache_gc:
-          gc_interval_sec: ${SPIDER_STORAGE_JOB_CACHE_GC_INTERVAL_SEC:-30}
-          terminated_job_retention_sec:
-            ${SPIDER_STORAGE_JOB_CACHE_TERMINATED_JOB_RETENTION_SEC:-300}
-        task_instance_pool:
-          execution_manager_stale_cutoff_sec:
-            ${SPIDER_STORAGE_TASK_INSTANCE_POOL_EXECUTION_MANAGER_STALE_CUTOFF_SEC:-60}
-          gc_interval_sec: ${SPIDER_STORAGE_TASK_INSTANCE_POOL_GC_INTERVAL_SEC:-30}
-          message_channel_capacity:
-            ${SPIDER_STORAGE_TASK_INSTANCE_POOL_MESSAGE_CHANNEL_CAPACITY:-128}
-  spider-worker-config:
-    content: |
-      connection_pool_size: ${SPIDER_WORKER_CONNECTION_POOL_SIZE:-4}
-      liveness:
-        scheduler_heartbeat_interval_sec: ${SPIDER_WORKER_SCHEDULER_HEARTBEAT_INTERVAL_SEC:-1}
-        storage_heartbeat_interval_sec: ${SPIDER_WORKER_STORAGE_HEARTBEAT_INTERVAL_SEC:-1}
-      scheduler:
-        host: "spider-scheduler"
-        port: ${SPIDER_SCHEDULER_PORT:-50052}
-      scheduler_poll_wait_ms: ${SPIDER_WORKER_SCHEDULER_POLL_WAIT_MS:-1000}
-      storage:
-        host: "spider-storage"
-        port: ${SPIDER_STORAGE_PORT:-50051}
-      task_executor:
-        bin_path: "${SPIDER_WORKER_TASK_EXECUTOR_BIN_PATH:-/usr/local/bin/spider-task-executor}"
-        inherited_env: ${SPIDER_WORKER_INHERITED_ENV:-[]}
-        max_log_line_bytes: ${SPIDER_WORKER_MAX_LOG_LINE_BYTES:-65536}
-        package_dir: "${SPIDER_WORKER_PACKAGE_DIR:-/opt/spider/packages}"
-
 networks:
   default:
   spider-internal:
diff --git a/tools/deployment/spider-compose/configs/scheduler.yaml b/tools/deployment/spider-compose/configs/scheduler.yaml
new file mode 100644
index 000000000..3832bd3b0
--- /dev/null
+++ b/tools/deployment/spider-compose/configs/scheduler.yaml
@@ -0,0 +1,30 @@
+configs:
+  spider-scheduler-config:
+    content: |
+      connection_pool_size: ${SPIDER_SCHEDULER_CONNECTION_POOL_SIZE:-8}
+      host: "0.0.0.0"
+      port: ${SPIDER_SCHEDULER_PORT:-50052}
+      runtime:
+        advertised_endpoint:
+          host: "spider-scheduler"
+          port: ${SPIDER_SCHEDULER_PORT:-50052}
+        em_registry:
+          dead_em_cutoff_sec: ${SPIDER_SCHEDULER_EM_REGISTRY_DEAD_EM_CUTOFF_SEC:-30}
+          liveness_tracking_interval_ms:
+            ${SPIDER_SCHEDULER_EM_REGISTRY_LIVENESS_TRACKING_INTERVAL_MS:-1000}
+        scheduler:
+          policy: "${SPIDER_SCHEDULER_POLICY:-round_robin}"
+          config:
+            active_job_queue_capacity: ${SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY:-16}
+            cleanup_ready_task_capacity: ${SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY:-256}
+            commit_ready_task_capacity: ${SPIDER_SCHEDULER_COMMIT_READY_TASK_CAPACITY:-256}
+            dispatch_queue_capacity: ${SPIDER_SCHEDULER_DISPATCH_QUEUE_CAPACITY:-16}
+            finalizing_job_expiration_timeout_sec:
+              ${SPIDER_SCHEDULER_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC:-300}
+            ready_task_capacity: ${SPIDER_SCHEDULER_READY_TASK_CAPACITY:-1048576}
+            storage_poll_timeout_ms: ${SPIDER_SCHEDULER_STORAGE_POLL_TIMEOUT_MS:-10}
+            tick_interval_ms: ${SPIDER_SCHEDULER_TICK_INTERVAL_MS:-5}
+        stop_timeout_sec: ${SPIDER_SCHEDULER_STOP_TIMEOUT_SEC:-30}
+      storage_endpoint:
+        host: "spider-storage"
+        port: ${SPIDER_STORAGE_PORT:-50051}
diff --git a/tools/deployment/spider-compose/configs/storage.yaml b/tools/deployment/spider-compose/configs/storage.yaml
new file mode 100644
index 000000000..0da3ba295
--- /dev/null
+++ b/tools/deployment/spider-compose/configs/storage.yaml
@@ -0,0 +1,25 @@
+configs:
+  spider-storage-config:
+    content: |
+      host: "0.0.0.0"
+      port: ${SPIDER_STORAGE_PORT:-50051}
+      runtime:
+        db:
+          host: "spider-database"
+          max_connections: ${SPIDER_STORAGE_DB_MAX_CONNECTIONS:-64}
+          name: "${SPIDER_DATABASE_NAME:-spider-db}"
+          port: ${SPIDER_DATABASE_PORT:-3306}
+        inbound_queue:
+          cleanup_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_CLEANUP_CAPACITY:-256}
+          commit_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_COMMIT_CAPACITY:-256}
+          task_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_TASK_CAPACITY:-1048576}
+        job_cache_gc:
+          gc_interval_sec: ${SPIDER_STORAGE_JOB_CACHE_GC_INTERVAL_SEC:-30}
+          terminated_job_retention_sec:
+            ${SPIDER_STORAGE_JOB_CACHE_TERMINATED_JOB_RETENTION_SEC:-300}
+        task_instance_pool:
+          execution_manager_stale_cutoff_sec:
+            ${SPIDER_STORAGE_TASK_INSTANCE_POOL_EXECUTION_MANAGER_STALE_CUTOFF_SEC:-60}
+          gc_interval_sec: ${SPIDER_STORAGE_TASK_INSTANCE_POOL_GC_INTERVAL_SEC:-30}
+          message_channel_capacity:
+            ${SPIDER_STORAGE_TASK_INSTANCE_POOL_MESSAGE_CHANNEL_CAPACITY:-128}
diff --git a/tools/deployment/spider-compose/configs/worker.yaml b/tools/deployment/spider-compose/configs/worker.yaml
new file mode 100644
index 000000000..f2ac69bf7
--- /dev/null
+++ b/tools/deployment/spider-compose/configs/worker.yaml
@@ -0,0 +1,19 @@
+configs:
+  spider-worker-config:
+    content: |
+      connection_pool_size: ${SPIDER_WORKER_CONNECTION_POOL_SIZE:-4}
+      liveness:
+        scheduler_heartbeat_interval_sec: ${SPIDER_WORKER_SCHEDULER_HEARTBEAT_INTERVAL_SEC:-1}
+        storage_heartbeat_interval_sec: ${SPIDER_WORKER_STORAGE_HEARTBEAT_INTERVAL_SEC:-1}
+      scheduler:
+        host: "spider-scheduler"
+        port: ${SPIDER_SCHEDULER_PORT:-50052}
+      scheduler_poll_wait_ms: ${SPIDER_WORKER_SCHEDULER_POLL_WAIT_MS:-1000}
+      storage:
+        host: "spider-storage"
+        port: ${SPIDER_STORAGE_PORT:-50051}
+      task_executor:
+        bin_path: "${SPIDER_WORKER_TASK_EXECUTOR_BIN_PATH:-/usr/local/bin/spider-task-executor}"
+        inherited_env: ${SPIDER_WORKER_INHERITED_ENV:-[]}
+        max_log_line_bytes: ${SPIDER_WORKER_MAX_LOG_LINE_BYTES:-65536}
+        package_dir: "${SPIDER_WORKER_PACKAGE_DIR:-/opt/spider/packages}"

From 78074363c0accc74218a5ae71c9ccbdfb8e90f86 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Wed, 12 Aug 2026 17:19:08 -0400
Subject: [PATCH 12/21] Rename env

---
 tools/deployment/spider-compose/.env.example  | 20 ++++++++---------
 .../spider-compose/configs/scheduler.yaml     | 22 ++++++++++++-------
 .../spider-compose/configs/worker.yaml        |  6 +++--
 3 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/tools/deployment/spider-compose/.env.example b/tools/deployment/spider-compose/.env.example
index 8b450765b..4ddb86d73 100644
--- a/tools/deployment/spider-compose/.env.example
+++ b/tools/deployment/spider-compose/.env.example
@@ -36,14 +36,14 @@ SPIDER_SCHEDULER_STOP_TIMEOUT_SEC=30
 SPIDER_SCHEDULER_EM_REGISTRY_DEAD_EM_CUTOFF_SEC=30
 SPIDER_SCHEDULER_EM_REGISTRY_LIVENESS_TRACKING_INTERVAL_MS=1000
 SPIDER_SCHEDULER_POLICY=round_robin
-SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY=16
-SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY=256
-SPIDER_SCHEDULER_COMMIT_READY_TASK_CAPACITY=256
-SPIDER_SCHEDULER_DISPATCH_QUEUE_CAPACITY=16
-SPIDER_SCHEDULER_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC=300
-SPIDER_SCHEDULER_READY_TASK_CAPACITY=1048576
-SPIDER_SCHEDULER_STORAGE_POLL_TIMEOUT_MS=10
-SPIDER_SCHEDULER_TICK_INTERVAL_MS=5
+SPIDER_SCHEDULER_ROUND_ROBIN_ACTIVE_JOB_QUEUE_CAPACITY=16
+SPIDER_SCHEDULER_ROUND_ROBIN_CLEANUP_READY_TASK_CAPACITY=256
+SPIDER_SCHEDULER_ROUND_ROBIN_COMMIT_READY_TASK_CAPACITY=256
+SPIDER_SCHEDULER_ROUND_ROBIN_DISPATCH_QUEUE_CAPACITY=16
+SPIDER_SCHEDULER_ROUND_ROBIN_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC=300
+SPIDER_SCHEDULER_ROUND_ROBIN_READY_TASK_CAPACITY=1048576
+SPIDER_SCHEDULER_ROUND_ROBIN_STORAGE_POLL_TIMEOUT_MS=10
+SPIDER_SCHEDULER_ROUND_ROBIN_TICK_INTERVAL_MS=5
 
 # Worker settings.
 # A complete worker image reference takes precedence over `SPIDER_IMAGE_TAG` when set.
@@ -51,9 +51,9 @@ SPIDER_SCHEDULER_TICK_INTERVAL_MS=5
 SPIDER_WORKER_REPLICAS=4
 SPIDER_WORKER_LOG_LEVEL=INFO
 SPIDER_WORKER_CONNECTION_POOL_SIZE=4
-SPIDER_WORKER_SCHEDULER_HEARTBEAT_INTERVAL_SEC=1
+SPIDER_WORKER_LIVENESS_SCHEDULER_HEARTBEAT_INTERVAL_SEC=1
 SPIDER_WORKER_SCHEDULER_POLL_WAIT_MS=1000
-SPIDER_WORKER_STORAGE_HEARTBEAT_INTERVAL_SEC=1
+SPIDER_WORKER_LIVENESS_STORAGE_HEARTBEAT_INTERVAL_SEC=1
 SPIDER_WORKER_TASK_EXECUTOR_BIN_PATH=/usr/local/bin/spider-task-executor
 SPIDER_WORKER_INHERITED_ENV=[]
 SPIDER_WORKER_MAX_LOG_LINE_BYTES=65536
diff --git a/tools/deployment/spider-compose/configs/scheduler.yaml b/tools/deployment/spider-compose/configs/scheduler.yaml
index 3832bd3b0..4c0728662 100644
--- a/tools/deployment/spider-compose/configs/scheduler.yaml
+++ b/tools/deployment/spider-compose/configs/scheduler.yaml
@@ -15,15 +15,21 @@ configs:
         scheduler:
           policy: "${SPIDER_SCHEDULER_POLICY:-round_robin}"
           config:
-            active_job_queue_capacity: ${SPIDER_SCHEDULER_ACTIVE_JOB_QUEUE_CAPACITY:-16}
-            cleanup_ready_task_capacity: ${SPIDER_SCHEDULER_CLEANUP_READY_TASK_CAPACITY:-256}
-            commit_ready_task_capacity: ${SPIDER_SCHEDULER_COMMIT_READY_TASK_CAPACITY:-256}
-            dispatch_queue_capacity: ${SPIDER_SCHEDULER_DISPATCH_QUEUE_CAPACITY:-16}
+            active_job_queue_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_ACTIVE_JOB_QUEUE_CAPACITY:-16}
+            cleanup_ready_task_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_CLEANUP_READY_TASK_CAPACITY:-256}
+            commit_ready_task_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_COMMIT_READY_TASK_CAPACITY:-256}
+            dispatch_queue_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_DISPATCH_QUEUE_CAPACITY:-16}
             finalizing_job_expiration_timeout_sec:
-              ${SPIDER_SCHEDULER_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC:-300}
-            ready_task_capacity: ${SPIDER_SCHEDULER_READY_TASK_CAPACITY:-1048576}
-            storage_poll_timeout_ms: ${SPIDER_SCHEDULER_STORAGE_POLL_TIMEOUT_MS:-10}
-            tick_interval_ms: ${SPIDER_SCHEDULER_TICK_INTERVAL_MS:-5}
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC:-300}
+            ready_task_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_READY_TASK_CAPACITY:-1048576}
+            storage_poll_timeout_ms:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_STORAGE_POLL_TIMEOUT_MS:-10}
+            tick_interval_ms: ${SPIDER_SCHEDULER_ROUND_ROBIN_TICK_INTERVAL_MS:-5}
         stop_timeout_sec: ${SPIDER_SCHEDULER_STOP_TIMEOUT_SEC:-30}
       storage_endpoint:
         host: "spider-storage"
diff --git a/tools/deployment/spider-compose/configs/worker.yaml b/tools/deployment/spider-compose/configs/worker.yaml
index f2ac69bf7..01537a429 100644
--- a/tools/deployment/spider-compose/configs/worker.yaml
+++ b/tools/deployment/spider-compose/configs/worker.yaml
@@ -3,8 +3,10 @@ configs:
     content: |
       connection_pool_size: ${SPIDER_WORKER_CONNECTION_POOL_SIZE:-4}
       liveness:
-        scheduler_heartbeat_interval_sec: ${SPIDER_WORKER_SCHEDULER_HEARTBEAT_INTERVAL_SEC:-1}
-        storage_heartbeat_interval_sec: ${SPIDER_WORKER_STORAGE_HEARTBEAT_INTERVAL_SEC:-1}
+        scheduler_heartbeat_interval_sec:
+          ${SPIDER_WORKER_LIVENESS_SCHEDULER_HEARTBEAT_INTERVAL_SEC:-1}
+        storage_heartbeat_interval_sec:
+          ${SPIDER_WORKER_LIVENESS_STORAGE_HEARTBEAT_INTERVAL_SEC:-1}
       scheduler:
         host: "spider-scheduler"
         port: ${SPIDER_SCHEDULER_PORT:-50052}

From 95f30cf4ab9f2238d73af7a3f35d9b71968f04b9 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Wed, 12 Aug 2026 21:20:25 -0400
Subject: [PATCH 13/21] Publish docker compose as oci artifact

---
 .github/workflows/spider-compose.yaml | 83 +++++++++++++++++++++++++++
 taskfiles/docker.yaml                 | 16 ++++++
 2 files changed, 99 insertions(+)
 create mode 100644 .github/workflows/spider-compose.yaml

diff --git a/.github/workflows/spider-compose.yaml b/.github/workflows/spider-compose.yaml
new file mode 100644
index 000000000..f4c552eea
--- /dev/null
+++ b/.github/workflows/spider-compose.yaml
@@ -0,0 +1,83 @@
+name: "spider-compose"
+
+on:
+  pull_request:
+    paths: &monitored_paths
+      - ".github/workflows/spider-compose.yaml"
+      - "taskfile.yaml"
+      - "taskfiles/docker.yaml"
+      - "tools/deployment/spider-compose/**"
+      - "tools/yscope-dev-utils"
+  push:
+    branches: ["main"]
+    paths: *monitored_paths
+    tags: ["spider-huntsman-v*"]
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: "${{github.workflow}}-${{github.ref}}"
+
+  # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted
+  # publishing of Docker Compose OCI artifacts.
+  cancel-in-progress: "${{github.ref != 'refs/heads/main'}}"
+
+jobs:
+  validate:
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"  # v6.0.2
+        with:
+          submodules: "recursive"
+
+      - uses: "./tools/yscope-dev-utils/exports/github/actions/install-go-task"
+
+      - name: "Validate Docker Compose configuration"
+        shell: "bash"
+        run: "task docker:compose:validate"
+
+  publish:
+    if: "github.event_name != 'pull_request'"
+    needs: "validate"
+    runs-on: "ubuntu-latest"
+    permissions:
+      contents: "read"
+      packages: "write"
+    steps:
+      - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"  # v6.0.2
+        with:
+          submodules: "recursive"
+
+      - uses: "./tools/yscope-dev-utils/exports/github/actions/install-go-task"
+
+      - uses: "oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d"  # v2.0.1
+        with:
+          version: "1.3.3"
+
+      - name: "Sanitize repository name"
+        id: "repo"
+        run: >-
+          echo "name=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
+
+      - name: "Get artifact tag"
+        id: "artifact"
+        shell: "bash"
+        run: |-
+          tag="nightly"
+          if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then
+            tag="${GITHUB_REF_NAME}"
+          fi
+          echo "tag=${tag}" >> "$GITHUB_OUTPUT"
+
+      - uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121"  # v4.1.0
+        with:
+          registry: "ghcr.io"
+          username: "${{github.actor}}"
+          password: "${{secrets.GITHUB_TOKEN}}"
+
+      - name: "Publish Docker Compose OCI artifact"
+        shell: "bash"
+        run: >-
+          task docker:compose:publish
+          OCI_REFERENCE='ghcr.io/${{steps.repo.outputs.name}}/spider-compose:${{steps.artifact.outputs.tag}}'
diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml
index 9fc187acb..787417d55 100644
--- a/taskfiles/docker.yaml
+++ b/taskfiles/docker.yaml
@@ -27,6 +27,22 @@ tasks:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"
     cmd: "docker compose --env-file .env.example --file compose.yaml config --quiet"
 
+  # Publishes the complete Docker Compose deployment directory as an OCI artifact.
+  #
+  # @param {string} OCI_REFERENCE The destination OCI reference, including its tag.
+  compose:publish:
+    requires:
+      vars: ["OCI_REFERENCE"]
+    deps: ["compose:validate"]
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}/.."
+    cmd: >-
+      oras push
+      --artifact-type "application/vnd.yscope.spider.compose.v1"
+      '{{.OCI_REFERENCE}}'
+      "spider-compose/compose.yaml:application/yaml"
+      "spider-compose/configs/:application/vnd.yscope.spider.compose.configs.v1.tar"
+      "spider-compose/.env.example:text/plain"
+
   compose:up:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"
     cmd: "docker compose --file compose.yaml up --detach --wait"

From 0ccb692e50713bf552bd44859d4feed27a532fa3 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Wed, 12 Aug 2026 21:27:05 -0400
Subject: [PATCH 14/21] Add 5 min wait timeout for docker compose up

---
 taskfiles/docker.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml
index 9fc187acb..6c834fd46 100644
--- a/taskfiles/docker.yaml
+++ b/taskfiles/docker.yaml
@@ -29,7 +29,7 @@ tasks:
 
   compose:up:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"
-    cmd: "docker compose --file compose.yaml up --detach --wait"
+    cmd: "docker compose --file compose.yaml up --detach --wait --wait-timeout 300"
 
   compose:down:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"

From 09ec07925a1b3f1981e9670e0503073439552ca7 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Wed, 12 Aug 2026 23:06:41 -0400
Subject: [PATCH 15/21] Change artifact name

---
 .github/workflows/spider-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spider-compose.yaml b/.github/workflows/spider-compose.yaml
index f4c552eea..cd366bce6 100644
--- a/.github/workflows/spider-compose.yaml
+++ b/.github/workflows/spider-compose.yaml
@@ -80,4 +80,4 @@ jobs:
         shell: "bash"
         run: >-
           task docker:compose:publish
-          OCI_REFERENCE='ghcr.io/${{steps.repo.outputs.name}}/spider-compose:${{steps.artifact.outputs.tag}}'
+          OCI_REFERENCE='ghcr.io/${{steps.repo.outputs.name}}/compose:${{steps.artifact.outputs.tag}}'

From 9729ac76cbf8d4dc719cfa8999f639e80befa9a8 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Thu, 13 Aug 2026 13:50:50 -0400
Subject: [PATCH 16/21] Only push to tag on tag creation

---
 .github/workflows/spider-compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/spider-compose.yaml b/.github/workflows/spider-compose.yaml
index cd366bce6..07c5f7919 100644
--- a/.github/workflows/spider-compose.yaml
+++ b/.github/workflows/spider-compose.yaml
@@ -65,7 +65,7 @@ jobs:
         shell: "bash"
         run: |-
           tag="nightly"
-          if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then
+          if [[ "${GITHUB_EVENT_NAME}" == "push" && "${GITHUB_REF_TYPE}" == "tag" ]]; then
             tag="${GITHUB_REF_NAME}"
           fi
           echo "tag=${tag}" >> "$GITHUB_OUTPUT"

From 152ab8ee6f31df438a5fece1d80ff0948992b206 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Fri, 14 Aug 2026 14:20:57 -0400
Subject: [PATCH 17/21] Remove media type version

---
 taskfiles/docker.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml
index 707f97474..79a92487a 100644
--- a/taskfiles/docker.yaml
+++ b/taskfiles/docker.yaml
@@ -37,10 +37,10 @@ tasks:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}/.."
     cmd: >-
       oras push
-      --artifact-type "application/vnd.yscope.spider.compose.v1"
+      --artifact-type "application/vnd.yscope.spider.compose"
       '{{.OCI_REFERENCE}}'
       "spider-compose/compose.yaml:application/yaml"
-      "spider-compose/configs/:application/vnd.yscope.spider.compose.configs.v1.tar"
+      "spider-compose/configs/:application/vnd.yscope.spider.compose.configs.tar"
       "spider-compose/.env.example:text/plain"
 
   compose:up:

From 43ede1f9e7954da4a1f4fb9fc1ef890c8412a747 Mon Sep 17 00:00:00 2001
From: sitaowang1998 <sitaowang1998@outlook.com>
Date: Fri, 14 Aug 2026 14:27:03 -0400
Subject: [PATCH 18/21] Apply suggestions from code review

Co-authored-by: ChenXing Yang <60459812+20001020ycx@users.noreply.github.com>
---
 .github/workflows/spider-compose.yaml | 5 ++++-
 taskfiles/docker.yaml                 | 3 ---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/spider-compose.yaml b/.github/workflows/spider-compose.yaml
index 07c5f7919..1ff6b7c65 100644
--- a/.github/workflows/spider-compose.yaml
+++ b/.github/workflows/spider-compose.yaml
@@ -45,7 +45,10 @@ jobs:
       contents: "read"
       packages: "write"
     steps:
-      - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"  # v6.0.2
+      - uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"  # v7.0.1
+        with:
+          persist-credentials: false
+          submodules: "recursive"
         with:
           submodules: "recursive"
 
diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml
index 707f97474..bc61a2e17 100644
--- a/taskfiles/docker.yaml
+++ b/taskfiles/docker.yaml
@@ -27,9 +27,6 @@ tasks:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"
     cmd: "docker compose --env-file .env.example --file compose.yaml config --quiet"
 
-  # Publishes the complete Docker Compose deployment directory as an OCI artifact.
-  #
-  # @param {string} OCI_REFERENCE The destination OCI reference, including its tag.
   compose:publish:
     requires:
       vars: ["OCI_REFERENCE"]

From a0dbcb5ba32d0ab158911435523f862599fbca13 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Fri, 14 Aug 2026 14:39:49 -0400
Subject: [PATCH 19/21] Add go task install version; Fix bug

---
 .github/workflows/spider-compose.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/spider-compose.yaml b/.github/workflows/spider-compose.yaml
index 1ff6b7c65..99c6faf33 100644
--- a/.github/workflows/spider-compose.yaml
+++ b/.github/workflows/spider-compose.yaml
@@ -32,6 +32,8 @@ jobs:
           submodules: "recursive"
 
       - uses: "./tools/yscope-dev-utils/exports/github/actions/install-go-task"
+        with:
+          version: "3.48.0"
 
       - name: "Validate Docker Compose configuration"
         shell: "bash"
@@ -49,10 +51,10 @@ jobs:
         with:
           persist-credentials: false
           submodules: "recursive"
-        with:
-          submodules: "recursive"
 
       - uses: "./tools/yscope-dev-utils/exports/github/actions/install-go-task"
+        with:
+          version: "3.48.0"
 
       - uses: "oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d"  # v2.0.1
         with:

From 3c8f3b097445d0901cada51916b4cb9eb3c16ee0 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Sun, 16 Aug 2026 23:10:38 -0400
Subject: [PATCH 20/21] Add workflow trigger every night

---
 .github/workflows/spider-compose.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/spider-compose.yaml b/.github/workflows/spider-compose.yaml
index 99c6faf33..e10460538 100644
--- a/.github/workflows/spider-compose.yaml
+++ b/.github/workflows/spider-compose.yaml
@@ -12,6 +12,9 @@ on:
     branches: ["main"]
     paths: *monitored_paths
     tags: ["spider-huntsman-v*"]
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
   workflow_dispatch:
 
 permissions: {}

From f49ccf5a7bf9a161a6220cc052b02a449f348997 Mon Sep 17 00:00:00 2001
From: Sitao Wang <sitaowang1998@outlook.com>
Date: Mon, 17 Aug 2026 11:47:10 -0400
Subject: [PATCH 21/21] Remove cron; Change default tag to main

---
 .github/workflows/spider-compose.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/spider-compose.yaml b/.github/workflows/spider-compose.yaml
index e10460538..6c31cac23 100644
--- a/.github/workflows/spider-compose.yaml
+++ b/.github/workflows/spider-compose.yaml
@@ -12,9 +12,6 @@ on:
     branches: ["main"]
     paths: *monitored_paths
     tags: ["spider-huntsman-v*"]
-  schedule:
-    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
-    - cron: "15 0 * * *"
   workflow_dispatch:
 
 permissions: {}
@@ -72,7 +69,7 @@ jobs:
         id: "artifact"
         shell: "bash"
         run: |-
-          tag="nightly"
+          tag="main"
           if [[ "${GITHUB_EVENT_NAME}" == "push" && "${GITHUB_REF_TYPE}" == "tag" ]]; then
             tag="${GITHUB_REF_NAME}"
           fi