[pull] main from Finsys:main

VERSION

@@ -1 +1 @@
-v1.0.48
+v1.0.49

package.json

@@ -1,7 +1,7 @@
 {
 	"name": "dockhand",
 	"private": true,
-	"version": "1.0.46",
+	"version": "1.0.49",
 	"type": "module",
 	"scripts": {
 		"dev": "npx vite dev",

src/lib/data/changelog.json

@@ -1,7 +1,7 @@
 [
   {
     "version": "1.0.49",
-    "date": "",
+    "date": "2026-09-22",
     "comingSoon": true,
     "changes": [
       { "type": "feature", "text": "tag containers and stacks, then filter and group the grid by tag (#781, #870)" },
@@ -26,7 +26,9 @@
       { "type": "feature", "text": "clear a search field with the X button in it (#1321)" },
       { "type": "fix", "text": "clearing a container's command now reverts it to the image default (#1489)" },
       { "type": "feature", "text": "one-shot container exec API returning stdout, stderr and exit code (#1380)" },
-      { "type": "feature", "text": "set and change file ownership in the container file browser (#224)" }
+      { "type": "feature", "text": "set and change file ownership in the container file browser (#224)" },
+      { "type": "fix", "text": "cut backup read transactions on S3/B2 by caching the restic index between runs (#1600)" },
+      { "type": "fix", "text": "dashboard no longer flashes zero running containers on a remote host (#1608)" }
     ],
     "imageTag": "fnsys/dockhand:v1.0.49"
   },

src/lib/openapi.generated.json

@@ -2,7 +2,7 @@
   "openapi": "3.0.3",
   "info": {
     "title": "Dockhand API",
-    "version": "1.0.46",
+    "version": "1.0.49",
     "description": "Auto-generated from src/routes/**/+server.ts by scripts/generate-openapi.ts. Path, HTTP method, tag, and auth requirement are derived automatically from the route tree and hooks.server.ts PUBLIC_PATHS. Parameters, response status codes, and request body fields are additionally auto-detected from each handler’s own source (query params via url.searchParams, status codes via status:/error(), body fields via destructuring) — adding a new endpoint therefore requires ZERO manual spec edits to show up with real params/responses. An optional, additive `@openapi` JSDoc annotation on a handler replaces the generic auto-descriptions with hand-written summaries, exact types, and examples."
   },
   "servers": [

src/lib/server/backups/restic-script.ts

@@ -10,6 +10,19 @@ import { isLocalRepo } from './models';
 /** The marker line the helper prints carrying restic's real exit code. */
 export const EXIT_MARKER = 'DOCKHAND_RESTIC_EXIT=';
 
+/**
+ * A persistent restic cache for the helper. The helper container is ephemeral
+ * (removed after each op), so a cache inside it is cold every run and restic
+ * re-downloads the whole repo index/snapshot metadata from the backend to dedup
+ * - many object GETs per backup, growing with the repo. A NAMED volume
+ * (not a host path) is managed by the TARGET daemon, so it survives the helper
+ * and works identically on local, direct-remote and Hawser envs - no host path
+ * is reachable on a remote daemon. restic keys its cache per-repo internally, so
+ * one shared volume serves every destination.
+ */
+export const RESTIC_CACHE_VOLUME = 'dockhand-restic-cache';
+export const RESTIC_CACHE_PATH = '/root/.cache/restic';
+
 /** Quote a single argument for safe interpolation into an `sh -c` string. */
 export function shellQuote(s: string): string {
 	return `'${s.replace(/'/g, `'\\''`)}'`;
@@ -160,6 +173,9 @@ export function buildHelperEnv(
 		`RESTIC_REPOSITORY=${repository}`,
 		`RESTIC_PASSWORD=${password}`,
 		'RESTIC_PROGRESS_FPS=2',
+		// Point restic at the mounted persistent cache (buildHelperBinds mounts the
+		// dockhand-restic-cache volume here) so it isn't cold on every run.
+		`RESTIC_CACHE_DIR=${RESTIC_CACHE_PATH}`,
 	];
 	for (const [k, v] of Object.entries(allowedCloudVars)) env.push(`${k}=${v}`);
 	// PEM CONTENT (not a path) - tlsCertPreamble writes it to a file in the helper.
@@ -178,7 +194,10 @@ export function buildHelperBinds(
 	volumeBinds: string[],
 	resolveLocalRepoHostPath: (repoPath: string) => string
 ): string[] {
-	const binds = [...volumeBinds];
+	// Persistent restic cache first (named volume, daemon-managed, survives the
+	// ephemeral helper) so backups reuse the repo index instead of re-fetching it
+	// from the backend every run.
+	const binds = [`${RESTIC_CACHE_VOLUME}:${RESTIC_CACHE_PATH}`, ...volumeBinds];
 	if (isLocalRepo(repository)) {
 		binds.push(`${resolveLocalRepoHostPath(repository)}:${repository}`);
 	}

src/lib/server/stacks.ts

@@ -2565,8 +2565,9 @@ export async function stopStack(
 
 	// `docker compose stop` interpolates the compose file too, so a stack with required
 	// provider secrets (${VAR:?required}) fails to stop without them - e.g. the
-	// stop-during-backup path (#1579).
-	await applyProviderSecretsToComposeResult(result, stackName, envId, `[Stack:${stackName}]`);
+	// stop-during-backup path (#1579). bestEffort: stopping a stack must not be blocked
+	// by an unreachable provider, so a resolve failure falls back to on-disk/DB vars.
+	await applyProviderSecretsToComposeResult(result, stackName, envId, `[Stack:${stackName}]`, true);
 
 	const composeResult = await executeComposeCommand(
 		'stop',
@@ -2622,8 +2623,10 @@ export async function restartStack(
 
 	// Resolve secret-provider values up front: every restart mode ends in a compose
 	// command (up/start/restart) that interpolates the compose file, so a stack with
-	// required provider secrets (${VAR:?required}) fails without them (#1579).
-	await applyProviderSecretsToComposeResult(result, stackName, envId, `[Stack:${stackName}]`);
+	// required provider secrets (${VAR:?required}) fails without them (#1579). bestEffort:
+	// restarting a running stack must not be blocked by an unreachable provider, so a
+	// resolve failure falls back to on-disk/DB vars rather than aborting the restart.
+	await applyProviderSecretsToComposeResult(result, stackName, envId, `[Stack:${stackName}]`, true);
 
 	if (mode === 'recreate') {
 		// Stop first, then bring up with --force-recreate to ensure new container IDs
@@ -2934,17 +2937,31 @@ export async function removeStack(
 		// "Remove stack" (deleteFiles=false) leaves the compose/.env/data on disk; "Remove
 		// stack + files" (default) deletes them.
 		if (stackDir && deleteFiles) {
-			try {
-				rmSync(stackDir, { recursive: true, force: true });
-			} catch (err: any) {
-				console.error(`Failed to delete stack directory: ${err.message}`);
-				cleanupErrors.push(`directory: ${err.message}`);
+			// The local `docker compose down` we just ran had this dir as its cwd
+			// (executeComposeCommand spawns with cwd: composeFileDir), so on a direct
+			// env the dir can be transiently busy the instant the process exits -
+			// rmSync then leaves it (EBUSY/ENOTEMPTY) or existsSync is still true.
+			// Retry a few times with a short backoff so the just-exited compose child
+			// releases its cwd/fds. Hawser envs run compose on the agent, so they
+			// never hit this.
+			let lastErr = '';
+			for (let attempt = 0; attempt < 5; attempt++) {
+				try {
+					rmSync(stackDir, { recursive: true, force: true });
+				} catch (err: any) {
+					lastErr = err.message;
+				}
+				if (!existsSync(stackDir)) { lastErr = ''; break; }
+				lastErr = lastErr || 'Directory still exists after deletion attempt';
+				await new Promise((r) => setTimeout(r, 200));
 			}
-			// Verify deletion succeeded (rmSync with force:true may not throw on some failures)
-			if (existsSync(stackDir)) {
-				const verifyErr = 'Directory still exists after deletion attempt';
-				console.error(`Failed to delete stack directory: ${verifyErr}`);
-				cleanupErrors.push(`directory: ${verifyErr}`);
+			if (lastErr) {
+				console.error(`Failed to delete stack directory: ${lastErr}`);
+				// A residual dir under Dockhand's own stacks tree blocks nothing when the
+				// caller forced removal: the containers are gone and a same-name recreate
+				// re-clones over it (syncGitStack rmSyncs the path first). So it is a
+				// warning under force, and only a hard failure (blocks recreate) otherwise.
+				cleanupErrors.push(`${force ? 'directory-warning' : 'directory'}: ${lastErr}`);
 			}
 		}
 
@@ -3765,10 +3782,11 @@ async function resolveProviderEnvVars(
 
 /**
  * Resolve the bound provider's secrets over a raw (nonSecretVars, secretVars) pair.
- * Returns the merged vars. bestEffort=true (tear-down paths) logs and returns the
- * inputs unchanged if the provider is unreachable, so a down/remove is never blocked
- * by a dead provider. The single source of truth both the compose-result helper and
- * removeStack use, so envPath derivation cannot drift between them.
+ * Returns the merged vars. bestEffort=true (lifecycle paths that must not be blocked by
+ * an unreachable provider: stop/restart/down/remove) logs and returns the inputs
+ * unchanged if the provider is down, so the op falls back to the on-disk/DB vars. The
+ * single source of truth both the compose-result helper and removeStack use, so envPath
+ * derivation cannot drift between them.
  */
 async function resolveProviderVarsBestEffort(
 	stackName: string,
@@ -3817,9 +3835,9 @@ async function applyProviderSecretsToComposeResult(
 	stackName: string,
 	envId: number | null | undefined,
 	logPrefix: string,
-	// Tear-down paths (down/remove) pass bestEffort: a provider that is unreachable
-	// must not block bringing a stack down, so a resolve error is logged and the
-	// compose command proceeds with whatever vars are on disk/DB.
+	// Lifecycle paths that must not be blocked by an unreachable provider (stop/restart/
+	// down/remove) pass bestEffort: a resolve error is logged and the compose command
+	// proceeds with whatever vars are on disk/DB. Deploy paths (start/up) keep the throw.
 	bestEffort = false
 ): Promise<void> {
 	if (!result.success || !result.secretVars || !result.nonSecretVars) return;

src/lib/utils/merge-partial-stats.ts

@@ -0,0 +1,55 @@
+/**
+ * Merge a progressive dashboard `partial` stats update into a tile's existing stats,
+ * IN PLACE (Svelte 5 reactivity relies on mutating the existing objects).
+ *
+ * The dashboard stats stream is re-opened periodically (the 30s refresh, a reconnect,
+ * a tab-visibility resume). Each re-open starts with a SKELETON partial: every data
+ * section (containers, images, volumes, ...) is a zeroed placeholder flagged
+ * `loading[section] = true`, filled in by a later partial. A naive deep-merge lets
+ * those placeholder zeros overwrite the real counts a populated tile already shows -
+ * and on a slow/remote env the follow-up partial can lag or be lost, leaving the tile
+ * stuck at zero even though the environment is healthy.
+ *
+ * Rule: skip a section's payload while its own `loading` flag is true (keep the prior
+ * value); still apply the `loading` object itself so the spinner shows. A section with
+ * no loading flag, or `loading:false`, is real data and is merged normally. On the very
+ * first load the tile's skeleton already shows zeros, so skipping the placeholder is a
+ * no-op there - the real partial (loading:false) fills it.
+ */
+export function mergePartialStats(
+	existing: Record<string, any>,
+	partial: Record<string, any>
+): void {
+	const loadingFlags = (partial.loading || {}) as Record<string, boolean>;
+	for (const [key, value] of Object.entries(partial)) {
+		if (value === undefined || key === 'id') continue;
+		if (loadingFlags[key] === true) continue; // skeleton placeholder - keep prior data
+		const current = existing[key];
+		if (
+			current && typeof current === 'object' && !Array.isArray(current) &&
+			value && typeof value === 'object' && !Array.isArray(value)
+		) {
+			Object.assign(current, value);
+		} else {
+			existing[key] = value;
+		}
+	}
+}
+
+/**
+ * The subset of a `partial` update that should reach the store, dropping placeholder
+ * sections (same rule as mergePartialStats) but always keeping `loading`. Returns a new
+ * object; does not mutate the input.
+ */
+export function definedPartialForStore(
+	partial: Record<string, any>
+): Record<string, any> {
+	const loadingFlags = (partial.loading || {}) as Record<string, boolean>;
+	const out: Record<string, any> = {};
+	for (const [key, value] of Object.entries(partial)) {
+		if (value === undefined) continue;
+		if (key !== 'loading' && loadingFlags[key] === true) continue;
+		out[key] = value;
+	}
+	return out;
+}

src/routes/+page.svelte

@@ -24,6 +24,7 @@
 	import { SearchInput } from '$lib/components/ui/search-input';
 	import MultiSelectFilter from '$lib/components/MultiSelectFilter.svelte';
 	import { appSettings } from '$lib/stores/settings';
+	import { mergePartialStats, definedPartialForStore } from '$lib/utils/merge-partial-stats';
 
 	const LABEL_FILTER_STORAGE_KEY = 'dockhand-dashboard-label-filter';
 
@@ -521,34 +522,18 @@
 										});
 									}
 								} else if (eventType === 'partial') {
-									// Progressive update - merge partial data into existing stats
-									// Use deep merge for nested objects to preserve existing values
+									// Progressive update - merge partial data into existing stats in place
+									// (Svelte 5 reactivity), skipping SKELETON placeholder sections so a
+									// re-opened stream's zeroed loading partial can't blank a populated tile.
 									const partialStats = data as Partial<EnvironmentStats> & { id: number };
 									const tile = tiles.find(t => t.id === partialStats.id);
 									if (tile?.stats) {
-										// Use direct mutation for Svelte 5 reactivity
-										// Deep merge for nested objects like containers, images, etc.
-										for (const [key, value] of Object.entries(partialStats)) {
-											if (value !== undefined && key !== 'id') {
-												const existing = (tile.stats as any)[key];
-												// Deep merge for plain objects (not arrays or null)
-												if (existing && typeof existing === 'object' && !Array.isArray(existing) &&
-												    value && typeof value === 'object' && !Array.isArray(value)) {
-													Object.assign(existing, value);
-												} else {
-													(tile.stats as any)[key] = value;
-												}
-											}
-										}
+										mergePartialStats(tile.stats as any, partialStats as any);
 									}
-									// Also update the store with deep merge
-									const definedStats: Partial<EnvironmentStats> = {};
-									for (const [key, value] of Object.entries(partialStats)) {
-										if (value !== undefined) {
-											(definedStats as any)[key] = value;
-										}
-									}
-									dashboardData.updateTilePartial(partialStats.id, definedStats);
+									dashboardData.updateTilePartial(
+										partialStats.id,
+										definedPartialForStore(partialStats as any) as Partial<EnvironmentStats>
+									);
 								} else if (eventType === 'stats') {
 									// Update the tile with actual stats (legacy/fallback)
 									const stats = data as EnvironmentStats;

static/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.0.3",
   "info": {
     "title": "Dockhand API",
-    "version": "1.0.46",
+    "version": "1.0.49",
     "description": "Auto-generated from src/routes/**/+server.ts by scripts/generate-openapi.ts. Path, HTTP method, tag, and auth requirement are derived automatically from the route tree and hooks.server.ts PUBLIC_PATHS. Parameters, response status codes, and request body fields are additionally auto-detected from each handler’s own source (query params via url.searchParams, status codes via status:/error(), body fields via destructuring) — adding a new endpoint therefore requires ZERO manual spec edits to show up with real params/responses. An optional, additive `@openapi` JSDoc annotation on a handler replaces the generic auto-descriptions with hand-written summaries, exact types, and examples."
   },
   "servers": [

tests/backups/restic-script.test.ts

@@ -11,6 +11,8 @@ import {
 	readExitMarker,
 	buildHelperEnv,
 	buildHelperBinds,
+	RESTIC_CACHE_VOLUME,
+	RESTIC_CACHE_PATH,
 	localRepoGuard,
 	localRepoChown,
 	classifyProcClose,
@@ -161,18 +163,23 @@ describe('binary stdout capture — Buffer.concat is lossless where string conca
 });
 
 describe('buildHelperEnv', () => {
-	it('sets repo, password, progress, plus allowlisted cloud vars', () => {
+	it('sets repo, password, progress, cache dir, plus allowlisted cloud vars', () => {
 		const env = buildHelperEnv('s3:bucket', 'pw', { AWS_ACCESS_KEY_ID: 'k' });
 		expect(env).toContain('RESTIC_REPOSITORY=s3:bucket');
 		expect(env).toContain('RESTIC_PASSWORD=pw');
 		expect(env).toContain('RESTIC_PROGRESS_FPS=2');
+		expect(env).toContain(`RESTIC_CACHE_DIR=${RESTIC_CACHE_PATH}`);
 		expect(env).toContain('AWS_ACCESS_KEY_ID=k');
 	});
+	it('points RESTIC_CACHE_DIR at the mounted cache volume path', () => {
+		const env = buildHelperEnv('s3:bucket', 'p', {});
+		expect(env).toContain(`RESTIC_CACHE_DIR=${RESTIC_CACHE_PATH}`);
+	});
 	it('is caller-filtered — it does NOT re-filter, so pass the allowlisted map', () => {
 		// The caller passes filterCloudEnvVars(...); this fn trusts that. Given an
 		// empty map, no cloud vars are added.
 		const env = buildHelperEnv('r', 'p', {});
-		expect(env).toHaveLength(3);
+		expect(env).toHaveLength(4);
 	});
 	it('adds the TLS PEM content vars when certs are supplied', () => {
 		const env = buildHelperEnv('rest:https://x/repo', 'p', {}, { cacert: 'CA-PEM', clientCert: 'CLIENT-PEM' });
@@ -193,13 +200,19 @@ describe('buildHelperEnv', () => {
 
 describe('buildHelperBinds', () => {
 	const resolve = (p: string) => `/host${p}`;
-	it('passes volume binds through unchanged for a remote repo', () => {
+	const cache = `${RESTIC_CACHE_VOLUME}:${RESTIC_CACHE_PATH}`;
+	it('prepends the persistent cache volume and passes volume binds through for a remote repo', () => {
 		const binds = buildHelperBinds('s3:bucket', ['vol:/volumes/vol:ro'], resolve);
-		expect(binds).toEqual(['vol:/volumes/vol:ro']);
+		expect(binds).toEqual([cache, 'vol:/volumes/vol:ro']);
 	});
-	it('adds the repo path bind for a local (filesystem) repo', () => {
+	it('adds the repo path bind for a local (filesystem) repo, cache still first', () => {
 		const binds = buildHelperBinds('/srv/restic', ['vol:/volumes/vol:ro'], resolve);
-		expect(binds).toEqual(['vol:/volumes/vol:ro', '/host/srv/restic:/srv/restic']);
+		expect(binds).toEqual([cache, 'vol:/volumes/vol:ro', '/host/srv/restic:/srv/restic']);
+	});
+	it('mounts the cache as a NAMED volume (no leading slash - daemon-managed, works on remote/hawser)', () => {
+		const binds = buildHelperBinds('s3:bucket', [], resolve);
+		expect(binds[0]).toBe(cache);
+		expect(binds[0].startsWith('/')).toBe(false);
 	});
 	it('does not mutate the caller array', () => {
 		const input = ['a:/volumes/a:ro'];