From 994e419c9d8c80313409c0cd4382d649c02e2071 Mon Sep 17 00:00:00 2001 From: Adam Eivy Date: Fri, 3 Jul 2026 07:25:33 -0700 Subject: [PATCH 1/5] feat([issue-2036]): re-render generated images from annotations (phase 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a user-triggered "Re-render with annotations" flow to the annotate page: flatten the source image + drawn strokes and feed the flattened PNG back through the existing local-FLUX img2img regen as the init image, so the marks reshape the render. Provider/model is named in a confirm dialog before any AI call runs (no cold-bootstrap), with optional prompt + strength. - server: /image-gen/:filename/regenerate gains `annotated` — seeds initImagePath from the saved sketch PNG sidecar (getSketchPngPath), higher default denoise, GPU-only (rejects the light method), 400 when no annotation saved. - client: rerenderWithAnnotations API + Re-render button/modal on MediaAnnotate, gated on getRegenAvailability; saves the annotation then queues the re-render. - tests: buildRegenParams annotated override, getSketchPngPath, the annotated route (no-annotation/ light / enqueue) paths, and the MediaAnnotate flow. Refs #2036 --- .changelog/NEXT.md | 5 + client/src/pages/MediaAnnotate.jsx | 158 +++++++++++++++++++++++- client/src/pages/MediaAnnotate.test.jsx | 108 ++++++++++++++++ client/src/services/apiImageVideo.js | 16 +++ server/routes/imageGen.js | 41 +++++- server/routes/imageGen.test.js | 70 +++++++++++ server/services/imageGen/regen.js | 18 ++- server/services/imageGen/regen.test.js | 20 +++ server/services/mediaSketches.js | 13 +- server/services/mediaSketches.test.js | 16 +++ 10 files changed, 453 insertions(+), 12 deletions(-) create mode 100644 .changelog/NEXT.md create mode 100644 client/src/pages/MediaAnnotate.test.jsx diff --git a/.changelog/NEXT.md b/.changelog/NEXT.md new file mode 100644 index 000000000..8cb6befd9 --- /dev/null +++ b/.changelog/NEXT.md @@ -0,0 +1,5 @@ +# Unreleased Changes + +## Media + +- **[issue-2036] Sketch & Annotation Canvas (phase 2): re-render an image guided by your annotations.** The annotate page (`/media/annotate/:mediaKey`) gained a "Re-render" action: draw over a generated image, then feed your markup back through local img2img so the marks reshape the render. A confirmation dialog names the exact local model it will run (no surprise AI calls) and lets you add an optional prompt and tune how much to change before it starts; the new render is queued and appears in Media History. Requires a local FLUX img2img runner — the action explains when one isn't available. Phase 3 (blank-canvas storyboard) remains open on #2036. diff --git a/client/src/pages/MediaAnnotate.jsx b/client/src/pages/MediaAnnotate.jsx index 91df71158..39b55fd21 100644 --- a/client/src/pages/MediaAnnotate.jsx +++ b/client/src/pages/MediaAnnotate.jsx @@ -1,11 +1,18 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { useParams, Link } from 'react-router-dom'; -import { Pencil, Eraser, Undo2, Trash2, Save, Download, ArrowLeft, ImageOff } from 'lucide-react'; +import { useParams, Link, useNavigate } from 'react-router-dom'; +import { Pencil, Eraser, Undo2, Trash2, Save, Download, ArrowLeft, ImageOff, Wand2, Loader2 } from 'lucide-react'; import toast from '../components/ui/Toast'; +import Modal from '../components/ui/Modal'; import AnnotationCanvas from '../components/media/AnnotationCanvas'; import { useAsyncAction } from '../hooks/useAsyncAction'; import { undoStrokes, clampSize, DEFAULT_COLOR, DEFAULT_SIZE, MIN_SIZE, MAX_SIZE } from '../lib/sketchCanvas'; -import { getMediaSketch, saveMediaSketch } from '../services/api'; +import { getMediaSketch, saveMediaSketch, getRegenAvailability, rerenderWithAnnotations } from '../services/api'; + +// Default denoise for an annotation re-render — high enough that the drawn marks +// actually reshape the image (mirrors the server's REGEN_ANNOTATED_STRENGTH_DEFAULT). +// Clamped into the backend's advertised [min, max] bounds at render time. +const DEFAULT_ANNOTATED_STRENGTH = 0.5; +const clamp = (n, lo, hi) => Math.min(hi, Math.max(lo, n)); // Parse a media key `:` on the client (mirrors server/lib/mediaItemKey.js // rules loosely — the server re-validates authoritatively). Phase 1 only supports @@ -24,6 +31,7 @@ const COLOR_SWATCHES = ['#ef4444', '#f59e0b', '#22c55e', '#3b82f6', '#a855f7', ' export default function MediaAnnotate() { const { mediaKey } = useParams(); + const navigate = useNavigate(); const parsed = useMemo(() => parseMediaKey(mediaKey), [mediaKey]); const isImage = parsed?.kind === 'image'; // Images are served from /data/images/; the ref IS the filename. @@ -38,6 +46,24 @@ export default function MediaAnnotate() { const [loading, setLoading] = useState(true); const [imageError, setImageError] = useState(false); + // Re-render (issue #2036 phase 2). `regenInfo` is the same local-FLUX + // img2img availability the lightbox uses — it names the exact model a + // re-render would run, so the user sees the provider/model before any AI + // call (no cold-bootstrap). null until fetched; `available:false` gates it. + const [regenInfo, setRegenInfo] = useState(null); + const [rerenderOpen, setRerenderOpen] = useState(false); + const [rerenderPrompt, setRerenderPrompt] = useState(''); + const [rerenderStrength, setRerenderStrength] = useState(DEFAULT_ANNOTATED_STRENGTH); + + useEffect(() => { + if (!isImage) return; + let active = true; + getRegenAvailability() + .then((r) => { if (active) setRegenInfo(r || null); }) + .catch(() => { if (active) setRegenInfo(null); }); + return () => { active = false; }; + }, [isImage]); + // Load any previously-saved strokes for this media key. React Router reuses // this component instance across :mediaKey changes (no remount), so reset all // transient view state synchronously before fetching — otherwise the previous @@ -94,6 +120,39 @@ export default function MediaAnnotate() { document.body.removeChild(a); }, [parsed]); + const strengthMin = regenInfo?.strengthMin ?? 0.02; + const strengthMax = regenInfo?.strengthMax ?? 0.6; + const regenAvailable = !!regenInfo?.available; + const regenModelLabel = regenInfo?.modelId || 'local FLUX img2img'; + + const openRerender = useCallback(() => { + setRerenderStrength(clamp(DEFAULT_ANNOTATED_STRENGTH, strengthMin, strengthMax)); + setRerenderPrompt(''); + setRerenderOpen(true); + }, [strengthMin, strengthMax]); + + // Persist the annotation (writes the flattened PNG sidecar the server uses as + // the img2img init image), then enqueue the re-render. Both API calls are + // silent so useAsyncAction owns the single error toast. + const [rerender, rerendering] = useAsyncAction(async () => { + if (!dims) throw new Error('Canvas not ready'); + if (strokes.length === 0) throw new Error('Draw over the image before re-rendering'); + const png = canvasApiRef.current?.exportPng?.() || undefined; + await saveMediaSketch(mediaKey, { width: dims.w, height: dims.h, strokes, png }, { silent: true }); + return rerenderWithAnnotations(parsed.ref, { + strength: rerenderStrength, + prompt: rerenderPrompt.trim() || undefined, + }); + }, { errorMessage: 'Failed to start re-render' }); + + const handleRerender = useCallback(async () => { + const res = await rerender(); + if (!res) return; + setRerenderOpen(false); + toast.success('Re-rendering with your annotations — it’ll appear in Media History'); + navigate('/media/history'); + }, [rerender, navigate]); + if (!isImage) { return (
@@ -208,11 +267,20 @@ export default function MediaAnnotate() { type="button" onClick={handleSave} disabled={saving || !dims} - className="px-3 py-1.5 text-xs bg-port-accent text-white rounded hover:bg-port-accent/80 disabled:opacity-40 inline-flex items-center gap-1" + className="px-3 py-1.5 text-xs bg-port-card border border-port-border text-gray-200 rounded hover:bg-port-border disabled:opacity-40 inline-flex items-center gap-1" title="Save annotation" > {saving ? 'Saving…' : 'Save'} +
@@ -240,6 +308,88 @@ export default function MediaAnnotate() { )} + + !rerendering && setRerenderOpen(false)} + size="sm" + closeOnBackdrop={!rerendering} + closeOnEsc={!rerendering} + ariaLabel="Re-render with annotations" + panelClassName="bg-port-card border border-port-border rounded-xl p-5 text-sm" + > +

+ Re-render with annotations +

+

+ Your drawing is flattened onto the image and fed back through img2img so the marks reshape the render. +

+ + {/* Provider/model visible before any AI call runs (no cold-bootstrap). */} +
+ Renders locally via + {regenModelLabel} +
+ + {!regenAvailable ? ( +
+ {regenInfo?.reason || 'Local img2img isn’t available on this install. A local FLUX runner is required to re-render.'} +
+ ) : ( + <> +
+ +