feat: Sketch & Annotation Canvas — phase 1 (annotate media images) (#2036)

.changelog/NEXT.md

@@ -1,5 +1,9 @@
 # Unreleased Changes
 
+## Media
+
+- **Sketch & Annotation Canvas (phase 1): draw over any generated image, save, reload, and export a flattened PNG.** Media History and Collection image cards gained an "Annotate" action that opens a new deep-linkable `/media/annotate/:mediaKey` page — a plain `<canvas>` stroke layer over the image with draw/erase, per-stroke color + brush-size, undo, and clear. Pointer events cover mouse, pen, and touch (`touch-action: none`, so drawing works at 360px phone width without scrolling the page). Strokes persist as a per-image sidecar (`data/media-sketches/`, JSON vectors stored in natural-pixel space so they restore exactly at any display size) plus a flattened PNG export, via a new Zod-validated `GET/PUT /api/media/sketches/:key` (retrievable at `/png`). Reachable from ⌘K and voice ("annotate"/"sketch"). Phases 2 (feed the markup back into img2img re-render) and 3 (blank-canvas storyboard) remain open on #2036.
+
 ## Dev Tools
 
 - **New interactive Integration Flows doc at `/devtools/flows`.** A single-page visual map of how PortOS packages talk to each other: ~60 components (client pages, client services, routes & sockets, server services, server lib, data stores, external systems) laid out in columns, with 10 documented end-to-end flows — Create Image, Start a Story, Voice Command, CoS Agent Run, Scheduled Task, Start/Stop App (PM2), Shell Session, Run Backup, Peer Sync, and Self-Update. Clicking a flow dims everything else, draws its numbered hops with transport-coded edges (HTTP / SSE / Socket.IO / EventEmitter / process spawn / file+DB I/O), and annotates what's passed at each hop; clicking a node shows its file, role, and the flows it participates in. The renderer (`client/public/flows.html`, dependency-free) is driven entirely by `client/public/flows.json`, so new flows are documented by editing JSON — an integrity test (`client/src/lib/flowsDoc.test.js`) verifies every referenced node/transport resolves and every referenced source file still exists, so refactors can't silently rot the doc. Launchable from the Dev Tools sidebar, ⌘K ("Flows"), and voice nav; selected flow is deep-linkable via `?flow=`, and "Open full screen" opens the standalone doc in a new tab.

client/src/App.jsx

@@ -42,6 +42,7 @@ const MediaGen = lazyWithReload(() => import('./pages/MediaGen'));
 const ImageGen = lazyWithReload(() => import('./pages/ImageGen'));
 const VideoGen = lazyWithReload(() => import('./pages/VideoGen'));
 const MediaHistory = lazyWithReload(() => import('./pages/MediaHistory'));
+const MediaAnnotate = lazyWithReload(() => import('./pages/MediaAnnotate'));
 const MediaCollections = lazyWithReload(() => import('./pages/MediaCollections'));
 const MediaCollectionDetail = lazyWithReload(() => import('./pages/MediaCollectionDetail'));
 const MediaCollectionSyncView = lazyWithReload(() => import('./pages/MediaCollectionSyncView'));
@@ -273,6 +274,8 @@ export default function App() {
             <Route path="image" element={<ImageGen />} />
             <Route path="video" element={<VideoGen />} />
             <Route path="history" element={<MediaHistory />} />
+            <Route path="annotate" element={<MediaAnnotate />} />
+            <Route path="annotate/:mediaKey" element={<MediaAnnotate />} />
             <Route path="collections" element={<MediaCollections />} />
             <Route path="collections/:id" element={<MediaCollectionDetail />} />
             <Route path="collections/:id/sync" element={<MediaCollectionSyncView />} />

client/src/components/media/AnnotationCanvas.jsx

@@ -0,0 +1,138 @@
+import { forwardRef, useCallback, useEffect, useImperativeHandle, useRef, useState } from 'react';
+import { drawStrokes, createStroke, appendPoint } from '../../lib/sketchCanvas';
+
+// Drawing surface for the Sketch & Annotation Canvas (issue #2036, phase 1).
+// Renders the target image with a transparent <canvas> stroke layer on top.
+// Controlled: the parent owns `strokes` + the active `tool` and receives
+// committed strokes via `onStrokesChange`. Pointer events cover mouse, pen, AND
+// touch (the canvas sets `touch-action: none` so a drag doesn't scroll the page).
+//
+// The imperative handle exposes `exportPng()` (flattened image + strokes) and
+// `dimensions` so the parent can persist / download without reaching into refs.
+const AnnotationCanvas = forwardRef(function AnnotationCanvas(
+  { imageSrc, strokes, tool, onStrokesChange, onImageLoad, onImageError },
+  ref,
+) {
+  const canvasRef = useRef(null);
+  const imgRef = useRef(null);
+  const drawingRef = useRef(null); // in-progress (uncommitted) stroke
+  const activePointerIdRef = useRef(null); // the single pointer we're tracking
+  const [dims, setDims] = useState(null); // { w, h } in natural pixels
+  const [inProgress, setInProgress] = useState(null);
+
+  const handleImgLoad = useCallback(() => {
+    const img = imgRef.current;
+    if (!img) return;
+    const w = img.naturalWidth || img.width || 1;
+    const h = img.naturalHeight || img.height || 1;
+    setDims({ w, h });
+    onImageLoad?.({ w, h });
+  }, [onImageLoad]);
+
+  // Redraw the whole layer from the committed strokes plus any in-progress
+  // stroke. Full redraw (rather than incremental) keeps undo/erase trivially
+  // correct — the layer is a pure function of the stroke list.
+  useEffect(() => {
+    const canvas = canvasRef.current;
+    if (!canvas || !dims) return;
+    const ctx = canvas.getContext('2d');
+    if (!ctx) return;
+    const all = inProgress ? [...strokes, inProgress] : strokes;
+    drawStrokes(ctx, all, canvas.width, canvas.height);
+  }, [strokes, inProgress, dims]);
+
+  // Map a pointer event to natural-pixel canvas coordinates, accounting for the
+  // CSS scale between the displayed size and the canvas's internal resolution.
+  const toNatural = useCallback((e) => {
+    const canvas = canvasRef.current;
+    const rect = canvas.getBoundingClientRect();
+    const scaleX = canvas.width / (rect.width || 1);
+    const scaleY = canvas.height / (rect.height || 1);
+    return {
+      x: (e.clientX - rect.left) * scaleX,
+      y: (e.clientY - rect.top) * scaleY,
+    };
+  }, []);
+
+  const handlePointerDown = useCallback((e) => {
+    // Track exactly one pointer — a second finger/palm mid-draw is ignored so
+    // it can't hijack or garble the in-progress stroke on touch devices.
+    if (!dims || activePointerIdRef.current !== null) return;
+    e.preventDefault();
+    activePointerIdRef.current = e.pointerId;
+    // Capture so strokes continue smoothly even when the pointer leaves the
+    // canvas bounds mid-draw. Auto-releases on pointerup/cancel.
+    canvasRef.current?.setPointerCapture?.(e.pointerId);
+    const { x, y } = toNatural(e);
+    const stroke = createStroke({ mode: tool.mode, color: tool.color, size: tool.size, x, y });
+    drawingRef.current = stroke;
+    setInProgress(stroke);
+  }, [dims, tool, toNatural]);
+
+  const handlePointerMove = useCallback((e) => {
+    if (e.pointerId !== activePointerIdRef.current || !drawingRef.current) return;
+    e.preventDefault();
+    const { x, y } = toNatural(e);
+    const next = appendPoint(drawingRef.current, x, y);
+    drawingRef.current = next;
+    setInProgress(next);
+  }, [toNatural]);
+
+  const finishStroke = useCallback((e) => {
+    if (e && e.pointerId !== activePointerIdRef.current) return;
+    activePointerIdRef.current = null;
+    const stroke = drawingRef.current;
+    if (!stroke) return;
+    drawingRef.current = null;
+    setInProgress(null);
+    onStrokesChange([...strokes, stroke]);
+  }, [strokes, onStrokesChange]);
+
+  useImperativeHandle(ref, () => ({
+    dimensions: dims,
+    // Flatten the image + stroke layer into a single PNG data URL at natural
+    // resolution. Same-origin image (/data/images/...) so the canvas isn't
+    // tainted and toDataURL succeeds.
+    exportPng: () => {
+      const canvas = canvasRef.current;
+      const img = imgRef.current;
+      if (!canvas || !img || !dims) return null;
+      const out = document.createElement('canvas');
+      out.width = dims.w;
+      out.height = dims.h;
+      const ctx = out.getContext('2d');
+      if (!ctx) return null;
+      ctx.drawImage(img, 0, 0, dims.w, dims.h);
+      ctx.drawImage(canvas, 0, 0, dims.w, dims.h);
+      return out.toDataURL('image/png');
+    },
+  }), [dims]);
+
+  return (
+    <div className="relative inline-block max-w-full bg-port-bg rounded-lg overflow-hidden">
+      <img
+        ref={imgRef}
+        src={imageSrc}
+        onLoad={handleImgLoad}
+        onError={onImageError}
+        alt="Media being annotated"
+        className="block max-w-full h-auto select-none"
+        draggable={false}
+      />
+      {dims && (
+        <canvas
+          ref={canvasRef}
+          width={dims.w}
+          height={dims.h}
+          onPointerDown={handlePointerDown}
+          onPointerMove={handlePointerMove}
+          onPointerUp={finishStroke}
+          onPointerCancel={finishStroke}
+          className="absolute inset-0 w-full h-full touch-none cursor-crosshair"
+        />
+      )}
+    </div>
+  );
+});
+
+export default AnnotationCanvas;

client/src/components/media/MediaCard.jsx

@@ -1,5 +1,5 @@
 import { useState } from 'react';
-import { Trash2, Download, Film, Image as ImageIcon, Sparkles, Eye, EyeOff, Maximize2, Wand2, Star, MessageSquare } from 'lucide-react';
+import { Trash2, Download, Film, Image as ImageIcon, Sparkles, Eye, EyeOff, Maximize2, Wand2, Star, MessageSquare, Pencil } from 'lucide-react';
 import MediaImage from '../MediaImage';
 import AddToCollectionMenu from './AddToCollectionMenu';
 import PinToMoodBoardMenu from './PinToMoodBoardMenu';
@@ -31,6 +31,7 @@ export default function MediaCard({
   starred = false,
   hasNote = false,
   onToggleStar,
+  onAnnotate,
 }) {
   const { kind, prompt, modelId, previewUrl, downloadUrl } = item;
   const isVideo = kind === 'video';
@@ -146,6 +147,17 @@ export default function MediaCard({
                 <Wand2 className="w-3 h-3" />
               </button>
             )}
+            {!isVideo && onAnnotate && (
+              <button
+                type="button"
+                onClick={() => onAnnotate(item)}
+                className="shrink-0 px-1.5 py-1 bg-port-accent/20 hover:bg-port-accent/40 text-port-accent text-[10px] rounded flex items-center justify-center"
+                title="Annotate (draw over this image)"
+                aria-label="Annotate image"
+              >
+                <Pencil className="w-3 h-3" />
+              </button>
+            )}
             {!isVideo && onSendToVideo && (
               <button
                 type="button"

client/src/lib/README.md

@@ -92,6 +92,7 @@ grep -i "what you want to do" client/src/lib/README.md
 | `mediaSearch.js` | `buildMediaHaystack`, `tokenizeQuery`, `matchHaystack`, `filterByQuery` — client-side AND-token search over normalized media items (prompt/model/seed/LoRA/universe tags). Shared by MediaHistory + the Image Gen gallery picker. |
 | `moodBoardItemSrc.js` | `moodBoardItemSrc(item)` resolves a mood-board item to a display image src (`imageUrl` → served `image:<file>` bytes → null). Shared by MoodBoardDetail + MoodBoardReferenceStrip. |
 | `sameJsonShape.js` | `sameJsonShape(prev, next)` — JSON.stringify-based equality for `useAutoRefetch`'s `compare` option on small, deterministically-shaped poll payloads. |
+| `sketchCanvas.js` | Pure stroke model + 2D-context renderer for the media annotation canvas (`createStroke`, `appendPoint`, `undoStrokes`, `drawStrokes`, `clampSize`). Points stored in natural-pixel space; erase strokes use `destination-out`. Used by `AnnotationCanvas.jsx` / `MediaAnnotate.jsx` (#2036). |
 | `unsorted.js` | Synthetic "Unsorted" collection from media not filed in any real collection. |
 | `upsertByIdPrepend.js` | Newest-first upsert into an id-keyed list. |
 | `voiceLabel.js` | `formatVoiceLabel(v, engine?)` — display label for a TTS voice record. Engine-specific formatters plug into a lookup table; new engines extend that map. |

client/src/lib/index.js

@@ -63,6 +63,7 @@ export * from './mediaNavigation.js';
 export * from './mediaSearch.js';
 export * from './moodBoardItemSrc.js';
 export * from './sameJsonShape.js';
+export * from './sketchCanvas.js';
 export * from './unsorted.js';
 export * from './upsertByIdPrepend.js';
 export * from './voiceLabel.js';

client/src/lib/sketchCanvas.js

@@ -0,0 +1,69 @@
+// Pure stroke model + 2D-context renderer for the media Sketch & Annotation
+// Canvas (issue #2036, phase 1). Kept free of React/DOM lookups so the drawing
+// and undo logic is unit-testable with a mock 2D context.
+//
+// Points are stored in the image's NATURAL-pixel space (the canvas element is
+// sized to naturalWidth x naturalHeight and CSS-scaled to fit). Storing natural
+// coordinates makes strokes restore identically at any display size — the phone
+// (360px) and desktop render the same persisted vectors.
+
+export const DEFAULT_COLOR = '#ef4444';
+export const DEFAULT_SIZE = 6;
+export const MIN_SIZE = 1;
+export const MAX_SIZE = 64;
+
+export const clampSize = (n) => {
+  const v = Math.round(Number(n));
+  if (!Number.isFinite(v)) return DEFAULT_SIZE;
+  return Math.max(MIN_SIZE, Math.min(MAX_SIZE, v));
+};
+
+export const createStroke = ({ mode = 'draw', color = DEFAULT_COLOR, size = DEFAULT_SIZE, x, y }) => ({
+  mode: mode === 'erase' ? 'erase' : 'draw',
+  color,
+  size: clampSize(size),
+  points: [{ x, y }],
+});
+
+// Immutable append — returns a new stroke so React state updates re-render.
+export const appendPoint = (stroke, x, y) => ({
+  ...stroke,
+  points: [...stroke.points, { x, y }],
+});
+
+// Pop the most recent committed stroke. Returns the same array reference when
+// empty so callers can no-op cheaply.
+export const undoStrokes = (strokes) => (strokes.length ? strokes.slice(0, -1) : strokes);
+
+// Render every stroke onto a 2D context sized `width` x `height`. Erase strokes
+// use destination-out so they cut transparent holes in earlier draw strokes
+// (revealing the underlying image when the layer is composited over it).
+export function drawStrokes(ctx, strokes, width, height) {
+  if (!ctx) return;
+  ctx.clearRect(0, 0, width, height);
+  for (const stroke of strokes) {
+    if (!stroke || !Array.isArray(stroke.points) || stroke.points.length === 0) continue;
+    ctx.save();
+    ctx.lineJoin = 'round';
+    ctx.lineCap = 'round';
+    ctx.lineWidth = stroke.size;
+    const erase = stroke.mode === 'erase';
+    ctx.globalCompositeOperation = erase ? 'destination-out' : 'source-over';
+    const paint = erase ? 'rgba(0,0,0,1)' : stroke.color;
+    ctx.strokeStyle = paint;
+    ctx.fillStyle = paint;
+    const pts = stroke.points;
+    if (pts.length === 1) {
+      // A single tap draws a dot rather than a zero-length line.
+      ctx.beginPath();
+      ctx.arc(pts[0].x, pts[0].y, Math.max(stroke.size / 2, 0.5), 0, Math.PI * 2);
+      ctx.fill();
+    } else {
+      ctx.beginPath();
+      ctx.moveTo(pts[0].x, pts[0].y);
+      for (let i = 1; i < pts.length; i += 1) ctx.lineTo(pts[i].x, pts[i].y);
+      ctx.stroke();
+    }
+    ctx.restore();
+  }
+}

client/src/lib/sketchCanvas.test.js

@@ -0,0 +1,84 @@
+import { describe, it, expect, vi } from 'vitest';
+import {
+  createStroke, appendPoint, undoStrokes, clampSize, drawStrokes,
+  DEFAULT_COLOR, DEFAULT_SIZE, MIN_SIZE, MAX_SIZE,
+} from './sketchCanvas';
+
+describe('sketchCanvas stroke model', () => {
+  it('clampSize bounds and rounds; falls back on garbage', () => {
+    expect(clampSize(1000)).toBe(MAX_SIZE);
+    expect(clampSize(-5)).toBe(MIN_SIZE);
+    expect(clampSize(6.7)).toBe(7);
+    expect(clampSize('abc')).toBe(DEFAULT_SIZE);
+  });
+
+  it('createStroke seeds a single point and normalizes mode/size', () => {
+    const s = createStroke({ mode: 'erase', color: '#fff', size: 9999, x: 2, y: 3 });
+    expect(s.mode).toBe('erase');
+    expect(s.color).toBe('#fff');
+    expect(s.size).toBe(MAX_SIZE);
+    expect(s.points).toEqual([{ x: 2, y: 3 }]);
+
+    const d = createStroke({ x: 0, y: 0 });
+    expect(d.mode).toBe('draw');
+    expect(d.color).toBe(DEFAULT_COLOR);
+    expect(d.size).toBe(DEFAULT_SIZE);
+  });
+
+  it('appendPoint returns a new stroke (immutable) with the point added', () => {
+    const s = createStroke({ x: 0, y: 0 });
+    const s2 = appendPoint(s, 5, 6);
+    expect(s2).not.toBe(s);
+    expect(s.points).toHaveLength(1); // original untouched
+    expect(s2.points).toEqual([{ x: 0, y: 0 }, { x: 5, y: 6 }]);
+  });
+
+  it('undoStrokes pops the last stroke, no-ops on empty', () => {
+    const a = createStroke({ x: 0, y: 0 });
+    const b = createStroke({ x: 1, y: 1 });
+    expect(undoStrokes([a, b])).toEqual([a]);
+    const empty = [];
+    expect(undoStrokes(empty)).toBe(empty);
+  });
+});
+
+describe('drawStrokes renderer', () => {
+  const makeCtx = () => ({
+    save: vi.fn(), restore: vi.fn(), beginPath: vi.fn(), moveTo: vi.fn(),
+    lineTo: vi.fn(), stroke: vi.fn(), arc: vi.fn(), fill: vi.fn(), clearRect: vi.fn(),
+    lineJoin: '', lineCap: '', lineWidth: 0, globalCompositeOperation: '', strokeStyle: '', fillStyle: '',
+  });
+
+  it('clears then draws a polyline for a multi-point draw stroke', () => {
+    const ctx = makeCtx();
+    const stroke = { mode: 'draw', color: '#f00', size: 4, points: [{ x: 0, y: 0 }, { x: 1, y: 1 }, { x: 2, y: 2 }] };
+    drawStrokes(ctx, [stroke], 10, 10);
+    expect(ctx.clearRect).toHaveBeenCalledWith(0, 0, 10, 10);
+    expect(ctx.moveTo).toHaveBeenCalledWith(0, 0);
+    expect(ctx.lineTo).toHaveBeenCalledTimes(2);
+    expect(ctx.stroke).toHaveBeenCalled();
+  });
+
+  it('draws a dot (arc+fill) for a single-point stroke', () => {
+    const ctx = makeCtx();
+    drawStrokes(ctx, [{ mode: 'draw', color: '#0f0', size: 8, points: [{ x: 3, y: 3 }] }], 10, 10);
+    expect(ctx.arc).toHaveBeenCalled();
+    expect(ctx.fill).toHaveBeenCalled();
+    expect(ctx.stroke).not.toHaveBeenCalled();
+  });
+
+  it('erase strokes use destination-out compositing', () => {
+    const ctx = makeCtx();
+    const composites = [];
+    // Record the composite op at the moment stroke() is invoked.
+    ctx.stroke = vi.fn(() => composites.push(ctx.globalCompositeOperation));
+    drawStrokes(ctx, [{ mode: 'erase', size: 5, points: [{ x: 0, y: 0 }, { x: 1, y: 1 }] }], 10, 10);
+    expect(composites).toContain('destination-out');
+  });
+
+  it('skips empty/invalid strokes without throwing', () => {
+    const ctx = makeCtx();
+    expect(() => drawStrokes(ctx, [null, { points: [] }, {}], 5, 5)).not.toThrow();
+    expect(ctx.stroke).not.toHaveBeenCalled();
+  });
+});