diff --git a/docs/source/tags/polygon.md b/docs/source/tags/polygon.md index f64c9852c902..cd159dd62af0 100644 --- a/docs/source/tags/polygon.md +++ b/docs/source/tags/polygon.md @@ -12,6 +12,10 @@ Use with the following data types: image. {% insertmd includes/tags/polygon.md %} +### Freehand drawing + +When the `fflag_feat_front_polygon_freehand` feature flag is enabled, select the Polygon tool and press and drag to draw a contour with a mouse, pen, or primary touch. The contour is simplified when the pointer is released and is saved as one undoable polygon. The flag is off by default; with it disabled, the existing click-to-place behavior is unchanged. + ### Example Basic labeling configuration for polygonal image segmentation diff --git a/web/libs/editor/src/components/ImageView/ImageView.jsx b/web/libs/editor/src/components/ImageView/ImageView.jsx index 7e9be8ca7770..730dfb250ee6 100644 --- a/web/libs/editor/src/components/ImageView/ImageView.jsx +++ b/web/libs/editor/src/components/ImageView/ImageView.jsx @@ -20,14 +20,23 @@ import ResizeObserver from "../../utils/resize-observer"; import { debounce } from "@humansignal/core/lib/utils/debounce"; import Constants from "../../core/Constants"; import { fixRectToFit, mapKonvaBrightness } from "../../utils/image"; -import { FF_DEV_1442, FF_LSDV_4930, FF_ZOOM_OPTIM, isFF } from "../../utils/feature-flags"; +import { FF_DEV_1442, FF_LSDV_4930, FF_POLYGON_FREEHAND, FF_ZOOM_OPTIM, isFF } from "../../utils/feature-flags"; import { Pagination } from "../../common/Pagination/Pagination"; import { Image } from "./Image"; +import { + appendFreehandPoint, + FREEHAND_MIN_DISTANCE, + FREEHAND_SIMPLIFY_EPSILON, + simplifyFreehandPoints, +} from "../../utils/freehand"; Konva.showWarnings = false; const hotkeys = Hotkey("Image"); const imgDefaultProps = { crossOrigin: "anonymous" }; +const FREEHAND_COMPATIBILITY_GUARD_MS = 1500; +const FREEHAND_COMPATIBILITY_GUARD_DISTANCE = 25; +const FREEHAND_DRAG_THRESHOLD = 5; export const splitRegions = (regions) => { const brushRegions = []; @@ -516,6 +525,7 @@ export default observer( state = { imgStyle: {}, pointer: [0, 0], + freehandPoints: [], }; imageRef = createRef(); @@ -527,6 +537,13 @@ export default observer( skipNextMouseUp = false; mouseDownPoint = null; mouseDown = false; + freehandPointerId = null; + freehandCaptureTarget = null; + freehandTool = null; + freehandTrace = []; + freehandDragging = false; + freehandCompatibilityGuard = null; + freehandCompatibilityClock = Date.now; constructor(props) { super(props); @@ -535,8 +552,252 @@ export default observer( props.store.settings.setSmoothing(props.item.smoothingEnabled); } + getFreehandTool = () => { + if (!isFF(FF_POLYGON_FREEHAND)) return null; + const tool = this.props.item.getToolsManager().findSelectedTool(); + + return tool?.toolName === "PolygonTool" && tool.canStartFreehand && tool.commitFreehand ? tool : null; + }; + + isRightElementToCatchToolInteractions = (element, isMoveTool) => { + // Bitmask is like Brush, so treat it the same. The only difference is + // that Bitmask doesn't have a group inside. + if (element.nodeType === "Layer" && !isMoveTool && element.attrs?.name === "bitmask") return true; + + if (element.nodeType === "Group") { + // It could be ruler or segmentation. + if (element.attrs?.name === "ruler") return true; + + // Segmentation is specific for Brushes, but click interaction on the + // region covers the same MoveTool interaction, so ignore MoveTool here + // to prevent conflicts. + if (!isMoveTool && element.attrs?.name === "segmentation") return true; + } + + return false; + }; + + isToolInteractionTarget = (target, isMoveTool = false) => { + const { item } = this.props; + + if (!target) return false; + return ( + item.getSkipInteractions() || + target === item.stageRef || + findClosestParent(target, (element) => this.isRightElementToCatchToolInteractions(element, isMoveTool)) + ); + }; + + getFreehandPoint = (event) => { + const { item } = this.props; + const stage = item.stageRef; + + if (!stage) return null; + if (Number.isFinite(event?.clientX) && Number.isFinite(event?.clientY)) stage.setPointersPositions(event); + + const screen = stage.getPointerPosition(); + + if (!screen || !Number.isFinite(screen.x) || !Number.isFinite(screen.y)) return null; + + const [canvasX, canvasY] = item.fixZoomedCoords([screen.x, screen.y]); + + return { + screen, + canvas: { x: canvasX, y: canvasY }, + internal: [item.canvasToInternalX(canvasX), item.canvasToInternalY(canvasY)], + }; + }; + + appendFreehandEventPoint = (event, force = false) => { + const point = this.getFreehandPoint(event); + + if (!point) return false; + + const nextTrace = appendFreehandPoint( + this.freehandTrace, + { ...point.screen, internal: point.internal }, + FREEHAND_MIN_DISTANCE, + force, + ); + + if (nextTrace === this.freehandTrace) return false; + this.freehandTrace = nextTrace; + if (!this.freehandDragging) { + const first = this.freehandTrace[0]; + const last = this.freehandTrace[this.freehandTrace.length - 1]; + const deltaX = last.x - first.x; + const deltaY = last.y - first.y; + + this.freehandDragging = deltaX * deltaX + deltaY * deltaY >= FREEHAND_DRAG_THRESHOLD * FREEHAND_DRAG_THRESHOLD; + } + this.setState(({ freehandPoints }) => ({ + freehandPoints: [...freehandPoints, point.canvas.x, point.canvas.y], + })); + return true; + }; + + attachFreehandFallback = () => { + window.addEventListener("pointermove", this.handlePointerMove, true); + window.addEventListener("pointerup", this.handlePointerUp, true); + window.addEventListener("pointercancel", this.handlePointerCancel, true); + }; + + detachFreehandFallback = () => { + window.removeEventListener("pointermove", this.handlePointerMove, true); + window.removeEventListener("pointerup", this.handlePointerUp, true); + window.removeEventListener("pointercancel", this.handlePointerCancel, true); + }; + + resetFreehand = ({ updateState = true } = {}) => { + const pointerId = this.freehandPointerId; + const captureTarget = this.freehandCaptureTarget; + + this.freehandPointerId = null; + this.freehandCaptureTarget = null; + this.freehandTool = null; + this.freehandTrace = []; + this.freehandDragging = false; + this.detachFreehandFallback(); + captureTarget?.removeEventListener?.("lostpointercapture", this.handleLostPointerCapture); + if (pointerId !== null && captureTarget?.hasPointerCapture?.(pointerId)) { + try { + captureTarget.releasePointerCapture(pointerId); + } catch { + // Synthetic events and capture-less browsers may not own native capture. + } + } + + if (updateState) this.setState({ freehandPoints: [] }); + }; + + rememberFreehandCompatibilityEvent = (event) => { + const clientX = Number.isFinite(event?.clientX) ? event.clientX : event?.offsetX; + const clientY = Number.isFinite(event?.clientY) ? event.clientY : event?.offsetY; + + if (!Number.isFinite(clientX) || !Number.isFinite(clientY)) return; + this.freehandCompatibilityGuard = { + clientX, + clientY, + expiresAt: this.freehandCompatibilityClock() + FREEHAND_COMPATIBILITY_GUARD_MS, + }; + }; + + shouldSuppressFreehandCompatibilityEvent = (event) => { + if (!isFF(FF_POLYGON_FREEHAND)) return false; + if (this.freehandDragging) return true; + + const guard = this.freehandCompatibilityGuard; + + if (!guard) return false; + if (this.freehandCompatibilityClock() >= guard.expiresAt) { + this.freehandCompatibilityGuard = null; + return false; + } + + const clientX = Number.isFinite(event?.clientX) ? event.clientX : event?.offsetX; + const clientY = Number.isFinite(event?.clientY) ? event.clientY : event?.offsetY; + const deltaX = clientX - guard.clientX; + const deltaY = clientY - guard.clientY; + + return ( + Number.isFinite(deltaX) && + Number.isFinite(deltaY) && + deltaX * deltaX + deltaY * deltaY <= + FREEHAND_COMPATIBILITY_GUARD_DISTANCE * FREEHAND_COMPATIBILITY_GUARD_DISTANCE + ); + }; + + handlePointerDown = (eventLike) => { + const event = eventLike.evt || eventLike; + + if (this.freehandPointerId !== null || event.isPrimary === false || (event.button ?? 0) !== 0) return; + + const tool = this.getFreehandTool(); + + if (!tool?.canStartFreehand()) return; + + const { item } = this.props; + + item.updateSkipInteractions(eventLike); + if (item.annotation.isReadOnly()) return; + if (eventLike.target?.getParent?.()?.className === "Transformer") return; + if (!this.isToolInteractionTarget(eventLike.target)) return; + + const point = this.getFreehandPoint(event); + + if (!point) return; + + this.freehandCompatibilityGuard = null; + this.freehandPointerId = event.pointerId; + this.freehandTool = tool; + this.freehandTrace = appendFreehandPoint([], { ...point.screen, internal: point.internal }); + this.setState({ freehandPoints: [point.canvas.x, point.canvas.y] }); + + const captureTarget = this.props.item.stageRef?.content; + + this.freehandCaptureTarget = captureTarget ?? null; + captureTarget?.addEventListener?.("lostpointercapture", this.handleLostPointerCapture); + try { + captureTarget?.setPointerCapture?.(event.pointerId); + } catch { + // Continue with window listeners when pointer capture is unavailable. + } + + if (!captureTarget?.hasPointerCapture?.(event.pointerId)) this.attachFreehandFallback(); + }; + + handlePointerMove = (eventLike) => { + const event = eventLike.evt || eventLike; + + if (event.pointerId !== this.freehandPointerId) return; + + const coalescedSamples = event.getCoalescedEvents?.(); + const samples = coalescedSamples?.length ? coalescedSamples : [event]; + + for (const sample of samples) this.appendFreehandEventPoint(sample); + if (this.freehandDragging) event.preventDefault?.(); + }; + + handlePointerUp = (eventLike) => { + const event = eventLike.evt || eventLike; + + if (event.pointerId !== this.freehandPointerId) return; + + this.appendFreehandEventPoint(event, true); + const wasDragging = this.freehandDragging; + + if (wasDragging) { + event.preventDefault?.(); + this.rememberFreehandCompatibilityEvent(event); + const points = simplifyFreehandPoints(this.freehandTrace, FREEHAND_SIMPLIFY_EPSILON).map( + ({ internal }) => internal, + ); + + if (points.length >= 3 && this.freehandTool === this.getFreehandTool() && isAlive(this.freehandTool)) { + this.freehandTool.commitFreehand(points); + } + } + + this.resetFreehand(); + }; + + handlePointerCancel = (eventLike) => { + const event = eventLike.evt || eventLike; + + if (event.pointerId !== this.freehandPointerId) return; + if (this.freehandDragging) this.rememberFreehandCompatibilityEvent(event); + this.resetFreehand(); + }; + + handleLostPointerCapture = (event) => { + if (event.pointerId === this.freehandPointerId) this.resetFreehand(); + }; + handleOnClick = (e) => { const { item } = this.props; + const evt = e.evt || e; + + if (this.shouldSuppressFreehandCompatibilityEvent(evt)) return; if (isFF(FF_DEV_1442)) { this.handleDeferredMouseDown?.(true); @@ -546,7 +807,6 @@ export default observer( return; } - const evt = e.evt || e; const { offsetX: x, offsetY: y } = evt; if (isFF(FF_LSDV_4930)) { @@ -617,6 +877,7 @@ export default observer( }; handleMouseDown = (e) => { + if (this.shouldSuppressFreehandCompatibilityEvent(e.evt || e)) return true; this.mouseDown = true; const { item } = this.props; const isPanTool = item.getToolsManager().findSelectedTool()?.fullName === "ZoomPanTool"; @@ -640,34 +901,7 @@ export default observer( e.evt.preventDefault(); } - const isRightElementToCatchToolInteractions = (el) => { - // Bitmask is like Brush, so treat it the same - // The only difference is that Bitmask doesn't have a group inside - if (el.nodeType === "Layer" && !isMoveTool && el.attrs?.name === "bitmask") { - return true; - } - - // It could be ruler ot segmentation - if (el.nodeType === "Group") { - if (el?.attrs?.name === "ruler") { - return true; - } - // segmentation is specific for Brushes - // but click interaction on the region covers the case of the same MoveTool interaction here, - // so it should ignore move tool interaction to prevent conflicts - if (!isMoveTool && el?.attrs?.name === "segmentation") { - return true; - } - } - return false; - }; - - if ( - // create regions over another regions with Cmd/Ctrl pressed - item.getSkipInteractions() || - e.target === item.stageRef || - findClosestParent(e.target, isRightElementToCatchToolInteractions) - ) { + if (this.isToolInteractionTarget(e.target, isMoveTool)) { window.addEventListener("mousemove", this.handleGlobalMouseMove); window.addEventListener("mouseup", this.handleGlobalMouseUp); const { offsetX: x, offsetY: y } = e.evt; @@ -753,6 +987,7 @@ export default observer( */ handleMouseUp = (e) => { this.mouseDown = false; + if (this.shouldSuppressFreehandCompatibilityEvent(e.evt || e)) return; const { item } = this.props; if (isFF(FF_DEV_1442)) { @@ -775,6 +1010,7 @@ export default observer( }; handleMouseMove = (e) => { + if (this.shouldSuppressFreehandCompatibilityEvent(e.evt || e)) return; const { item } = this.props; item.freezeHistory(); @@ -984,6 +1220,7 @@ export default observer( }; componentWillUnmount() { + this.resetFreehand({ updateState: false }); this.detachObserver(); window.removeEventListener("resize", this.onResize); window.removeEventListener("mousemove", this.handleGlobalMouseMove); @@ -1152,6 +1389,10 @@ export default observer( onMouseDown={this.handleMouseDown} onMouseMove={this.handleMouseMove} onMouseUp={this.handleMouseUp} + onPointerDown={this.handlePointerDown} + onPointerMove={this.handlePointerMove} + onPointerUp={this.handlePointerUp} + onPointerCancel={this.handlePointerCancel} onWheel={item.zoom ? this.handleZoom : () => {}} /> ) : null} @@ -1191,10 +1432,16 @@ const EntireStage = observer( onMouseDown, onMouseMove, onMouseUp, + onPointerDown, + onPointerMove, + onPointerUp, + onPointerCancel, onWheel, crosshairRef, }) => { const { store } = item; + const selectedTool = item.getToolsManager().findSelectedTool(); + const freehandEnabled = isFF(FF_POLYGON_FREEHAND) && selectedTool?.toolName === "PolygonTool"; let size; let position; @@ -1220,7 +1467,13 @@ const EntireStage = observer( ref={(ref) => { item.setStageRef(ref); }} - className={[styles["image-element"], ...imagePositionClassnames].join(" ")} + className={[ + styles["image-element"], + ...imagePositionClassnames, + freehandEnabled ? styles["freehand-enabled"] : null, + ] + .filter(Boolean) + .join(" ")} width={size.width} height={size.height} scaleX={item.zoomScale} @@ -1237,6 +1490,10 @@ const EntireStage = observer( onMouseDown={onMouseDown} onMouseMove={onMouseMove} onMouseUp={onMouseUp} + onPointerDown={onPointerDown} + onPointerMove={onPointerMove} + onPointerUp={onPointerUp} + onPointerCancel={onPointerCancel} onWheel={onWheel} > @@ -1440,6 +1697,19 @@ const StageContent = observer(({ item, store, state, crosshairRef }) => { ); })} + {state.freehandPoints.length >= 4 && ( + + + + )} {item.smoothingEnabled === false && } diff --git a/web/libs/editor/src/components/ImageView/ImageView.module.css b/web/libs/editor/src/components/ImageView/ImageView.module.css index 0fabef1a3319..2a644cb1bc3d 100644 --- a/web/libs/editor/src/components/ImageView/ImageView.module.css +++ b/web/libs/editor/src/components/ImageView/ImageView.module.css @@ -52,6 +52,10 @@ position: absolute; } +.freehand-enabled { + touch-action: none; +} + .image_position { position: absolute; diff --git a/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx b/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx index 63f942a61a27..4a91ce824a14 100644 --- a/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx +++ b/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx @@ -2,13 +2,14 @@ * Unit tests for ImageView (components/ImageView/ImageView.jsx) */ import React from "react"; -import { render, screen, fireEvent } from "@testing-library/react"; +import { act, render, screen, fireEvent } from "@testing-library/react"; import ImageView, { splitRegions } from "../ImageView"; jest.mock("../../../utils/feature-flags", () => ({ isFF: jest.fn(() => false), FF_DEV_1442: "fflag_dev_1442", FF_LSDV_4930: "fflag_lsdv_4930", + FF_POLYGON_FREEHAND: "fflag_feat_front_polygon_freehand", FF_ZOOM_OPTIM: "fflag_zoom_optim", })); @@ -207,6 +208,57 @@ function createStore(overrides = {}) { }; } +function createFreehandPointerHarness() { + const store = createStore(); + const commitFreehand = jest.fn(); + const tool = { + toolName: "PolygonTool", + canStartFreehand: jest.fn(() => true), + commitFreehand, + }; + let pointerPosition = { x: 10, y: 10 }; + let transformOffset = 0; + const captureTarget = { + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + setPointerCapture: jest.fn(), + hasPointerCapture: jest.fn(() => true), + releasePointerCapture: jest.fn(), + }; + const stageRef = { + content: captureTarget, + getPointerPosition: () => pointerPosition, + setPointersPositions: (event) => { + pointerPosition = { x: event.clientX, y: event.clientY }; + }, + on: jest.fn(), + off: jest.fn(), + getStage: () => null, + position: () => ({ x: 0, y: 0 }), + scale: () => ({ x: 1, y: 1 }), + }; + const item = createItem({ + stageRef, + getToolsManager: () => ({ findSelectedTool: () => tool, allTools: () => [tool] }), + fixZoomedCoords: ([x, y]) => [x + transformOffset, y + transformOffset], + canvasToInternalX: (x) => x, + canvasToInternalY: (y) => y, + }); + item.store = store; + let viewRef; + + render( (viewRef = ref)} item={item} store={store} />); + return { + commitFreehand, + item, + stageRef, + setTransformOffset: (offset) => { + transformOffset = offset; + }, + view: () => viewRef, + }; +} + describe("splitRegions", () => { it("returns empty arrays for empty regions", () => { const result = splitRegions([]); @@ -1222,4 +1274,215 @@ describe("ImageView with feature flags", () => { expect(item.event).toHaveBeenCalledWith("mousedown", expect.anything(), 20, 30); jest.useRealTimers(); }); + + it("creates points from a pointer drag only when freehand drawing is enabled", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, stageRef, view } = createFreehandPointerHarness(); + const drag = () => { + act(() => { + view().handlePointerDown({ + target: stageRef, + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); + view().handlePointerMove({ + pointerId: 1, + clientX: 40, + clientY: 10, + getCoalescedEvents: () => [], + preventDefault: jest.fn(), + }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 10, clientY: 40, preventDefault: jest.fn() }); + }); + }; + + isFF.mockReturnValue(false); + drag(); + expect(commitFreehand).not.toHaveBeenCalled(); + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + drag(); + expect(commitFreehand).toHaveBeenCalledTimes(1); + expect(commitFreehand.mock.calls[0][0]).toHaveLength(4); + }); + + it("keeps each sample in the coordinate transform active when it was collected", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, setTransformOffset, stageRef, view } = createFreehandPointerHarness(); + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + act(() => { + view().handlePointerDown({ + target: stageRef, + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 10, preventDefault: jest.fn() }); + setTransformOffset(100); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 10, clientY: 40, preventDefault: jest.fn() }); + }); + + expect(commitFreehand).toHaveBeenCalledWith([ + [10, 10], + [40, 10], + [140, 140], + [110, 140], + ]); + }); + + it("lets a below-threshold pointer gesture continue through the stock click path", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, item, stageRef, view } = createFreehandPointerHarness(); + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + act(() => { + view().handlePointerDown({ + target: stageRef, + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); + view().handlePointerMove({ pointerId: 1, clientX: 13, clientY: 10, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 13, clientY: 10, preventDefault: jest.fn() }); + }); + + expect(commitFreehand).not.toHaveBeenCalled(); + expect(view().shouldSuppressFreehandCompatibilityEvent({ clientX: 13, clientY: 10 })).toBe(false); + view().handleOnClick({ evt: { offsetX: 13, offsetY: 10, clientX: 13, clientY: 10 } }); + expect(item.event).toHaveBeenCalledWith("click", expect.anything(), 13, 10); + }); + + it("prunes a compatibility guard after its TTL", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { view } = createFreehandPointerHarness(); + let now = 1000; + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + view().freehandCompatibilityClock = () => now; + view().rememberFreehandCompatibilityEvent({ clientX: 10, clientY: 20 }); + + expect(view().freehandCompatibilityGuard.expiresAt).toBe(2500); + now = 2499; + expect(view().shouldSuppressFreehandCompatibilityEvent({ clientX: 10, clientY: 20 })).toBe(true); + now = 2501; + expect(view().shouldSuppressFreehandCompatibilityEvent({ clientX: 10, clientY: 20 })).toBe(false); + expect(view().freehandCompatibilityGuard).toBeNull(); + }); + + it("does not start freehand drawing from an existing region or transformer", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, view } = createFreehandPointerHarness(); + const dragFrom = (target, pointerId) => { + act(() => { + view().handlePointerDown({ + target, + pointerId, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); + view().handlePointerMove({ pointerId, clientX: 40, clientY: 10, preventDefault: jest.fn() }); + view().handlePointerMove({ pointerId, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId, clientX: 10, clientY: 40, preventDefault: jest.fn() }); + }); + }; + const region = { parent: null, getParent: () => null }; + const transformer = { className: "Transformer" }; + const anchor = { parent: transformer, getParent: () => transformer }; + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + dragFrom(region, 1); + dragFrom(anchor, 2); + + expect(commitFreehand).not.toHaveBeenCalled(); + expect(view().freehandPointerId).toBeNull(); + }); + + it("does not start freehand drawing on a read-only annotation", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, item, stageRef, view } = createFreehandPointerHarness(); + + item.annotation.isReadOnly = () => true; + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + act(() => { + view().handlePointerDown({ + target: stageRef, + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 10, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + }); + + expect(commitFreehand).not.toHaveBeenCalled(); + expect(view().freehandPointerId).toBeNull(); + }); + + it("allows freehand drawing over a region when Cmd/Ctrl interaction skipping is active", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, item, view } = createFreehandPointerHarness(); + const region = { parent: null, getParent: () => null }; + let skipInteractions = false; + + item.updateSkipInteractions = jest.fn(({ evt }) => { + skipInteractions = evt.metaKey || evt.ctrlKey; + }); + item.getSkipInteractions = () => skipInteractions; + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + act(() => { + view().handlePointerDown({ + target: region, + evt: { + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + metaKey: true, + }, + }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 10, preventDefault: jest.fn() }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 10, clientY: 40, preventDefault: jest.fn() }); + }); + + expect(item.updateSkipInteractions).toHaveBeenCalledTimes(1); + expect(commitFreehand).toHaveBeenCalledTimes(1); + }); + + it("enables touch-action suppression only while Polygon is selected", () => { + const { isFF } = require("../../../utils/feature-flags"); + const store = createStore(); + const polygonTool = { toolName: "PolygonTool" }; + const polygonItem = createItem({ + getToolsManager: () => ({ findSelectedTool: () => polygonTool, allTools: () => [polygonTool] }), + }); + const rectangleTool = { toolName: "RectangleTool" }; + const rectangleItem = createItem({ + getToolsManager: () => ({ findSelectedTool: () => rectangleTool, allTools: () => [rectangleTool] }), + }); + + polygonItem.store = store; + rectangleItem.store = store; + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + const { rerender } = render(); + expect(screen.getByTestId("konva-stage").className).toContain("freehand-enabled"); + + rerender(); + expect(screen.getByTestId("konva-stage").className).not.toContain("freehand-enabled"); + }); }); diff --git a/web/libs/editor/src/tools/Polygon.js b/web/libs/editor/src/tools/Polygon.js index b782909715da..df9faa211dab 100644 --- a/web/libs/editor/src/tools/Polygon.js +++ b/web/libs/editor/src/tools/Polygon.js @@ -6,6 +6,9 @@ import ToolMixin from "../mixins/Tool"; import { MultipleClicksDrawingTool } from "../mixins/DrawingTool"; import { NodeViews } from "../components/Node/Node"; import { observe } from "mobx"; +import { FF_POLYGON_FREEHAND, isFF } from "../utils/feature-flags"; + +const FREEHAND_HISTORY_KEY = "polygon-freehand"; const _Tool = types .model("PolygonTool", { @@ -75,10 +78,76 @@ const _Tool = types .actions((self) => { let disposer; let closed; + let freehandHistoryFrozen = false; + let freehandFinishTimer = null; + + const releaseFreehandHistory = () => { + if (!freehandHistoryFrozen) return; + freehandHistoryFrozen = false; + self.annotation.history.unfreeze(FREEHAND_HISTORY_KEY); + }; + + const finishFreehandDrawing = () => { + if (!self.isDrawing || !self.getCurrentArea()) { + releaseFreehandHistory(); + return false; + } + + self.annotation.regionStore.selection.drawingUnselect(); + self.closeCurrent(); + freehandFinishTimer = setTimeout(() => { + freehandFinishTimer = null; + if (!isAlive(self) || !self.isDrawing || !self.annotation.isDrawing || !self.getCurrentArea()) { + releaseFreehandHistory(); + return; + } + self._finishDrawing(); + }); + return true; + }; return { + canStartFreehand() { + return isFF(FF_POLYGON_FREEHAND) && self.canStartDrawing(); + }, + commitFreehand(points) { + if (!Array.isArray(points) || points.length < 3 || !self.canStartFreehand()) return false; + + const validPoints = points.filter( + (point) => Array.isArray(point) && Number.isFinite(point[0]) && Number.isFinite(point[1]), + ); + + if (validPoints.length < 3) return false; + + self.stopListening(); + closed = false; + self.annotation.history.freeze(FREEHAND_HISTORY_KEY); + freehandHistoryFrozen = true; + try { + self.startDrawing(validPoints[0][0], validPoints[0][1]); + + if (!self.isDrawing || !self.getCurrentArea()) { + releaseFreehandHistory(); + return false; + } + + validPoints.slice(1).forEach(([x, y]) => self.nextPoint(x, y)); + if (self.getCurrentArea().points.length < 3) { + self.cleanupUncloseableShape(); + releaseFreehandHistory(); + return false; + } + + return finishFreehandDrawing(); + } catch (error) { + releaseFreehandHistory(); + throw error; + } + }, handleToolSwitch(tool) { self.stopListening(); + releaseFreehandHistory(); + if (freehandFinishTimer !== null) return; if (self.getCurrentArea()?.isDrawing && tool.toolName !== "ZoomPanTool") { const shape = self.getCurrentArea()?.toJSON(); @@ -100,7 +169,16 @@ const _Tool = types ); }, stopListening() { - if (disposer) disposer(); + if (disposer) { + disposer(); + disposer = null; + } + }, + beforeDestroy() { + self.stopListening(); + if (freehandFinishTimer !== null) clearTimeout(freehandFinishTimer); + freehandFinishTimer = null; + releaseFreehandHistory(); }, closeCurrent() { self.stopListening(); @@ -121,13 +199,17 @@ const _Tool = types }, _finishDrawing() { - const { currentArea, control } = self; - - self.currentArea.notifyDrawingFinished(); - self.setDrawing(false); - self.currentArea = null; - self.mode = "viewing"; - self.annotation.afterCreateResult(currentArea, control); + try { + const { currentArea, control } = self; + + self.currentArea.notifyDrawingFinished(); + self.setDrawing(false); + self.currentArea = null; + self.mode = "viewing"; + self.annotation.afterCreateResult(currentArea, control); + } finally { + releaseFreehandHistory(); + } }, setDrawing(drawing) { diff --git a/web/libs/editor/src/tools/__tests__/Polygon.test.js b/web/libs/editor/src/tools/__tests__/Polygon.test.js new file mode 100644 index 000000000000..17e414fb40ea --- /dev/null +++ b/web/libs/editor/src/tools/__tests__/Polygon.test.js @@ -0,0 +1,177 @@ +let mockFreehandEnabled = false; + +jest.mock("../../utils/feature-flags", () => ({ + FF_DEV_3391: "fflag_fix_front_dev_3391_interactive_view_all", + FF_SIMPLE_INIT: "fflag_fix_front_leap_443_select_annotation_once", + FF_POLYGON_FREEHAND: "fflag_feat_front_polygon_freehand", + isFF: jest.fn((flag) => { + if (flag === "fflag_feat_front_polygon_freehand") return mockFreehandEnabled; + return flag === "fflag_fix_front_dev_3391_interactive_view_all"; + }), +})); + +jest.mock("@humansignal/core", () => ({ + ff: { + FF_MULTIPLE_LABELS_REGIONS: "fflag_multiple_labels_regions", + isActive: jest.fn(() => false), + }, +})); + +jest.mock("../../components/Node/Node", () => ({ + NodeViews: { PolygonRegionModel: { icon: jest.fn(), altIcon: jest.fn() } }, +})); + +const { Polygon } = require("../Polygon"); + +const createPolygonTool = () => { + const history = { freeze: jest.fn(), unfreeze: jest.fn() }; + const selection = { drawingUnselect: jest.fn(), hasSelection: false }; + const annotation = { + editable: true, + isDrawing: false, + isReadOnly: jest.fn(() => false), + history, + regionStore: { selection, hasSelection: false }, + setIsDrawing: jest.fn((drawing) => { + annotation.isDrawing = drawing; + }), + afterCreateResult: jest.fn(), + unselectAll: jest.fn(), + }; + const control = { + type: "polygonlabels", + isSelected: true, + isSeparated: false, + annotation, + getResultValue: jest.fn(() => ({})), + getSnappedPoint: jest.fn(({ x, y }) => ({ x, y })), + }; + const area = { + type: "polygonregion", + closed: false, + isDrawing: true, + points: [], + setValue: jest.fn(), + setDrawing: jest.fn((drawing) => { + area.isDrawing = drawing; + }), + addPoint: jest.fn((x, y) => area.points.push({ x, y })), + closePoly: jest.fn(() => { + area.closed = true; + }), + notifyDrawingFinished: jest.fn(), + toJSON: jest.fn(() => ({ points: area.points.map(({ x, y }) => [x, y]) })), + }; + annotation.createResult = jest.fn((options) => { + area.points = options.points.map(([x, y]) => ({ x, y })); + return area; + }); + const object = { + name: "image", + annotation, + regs: [], + canvasSize: { width: 100, height: 100 }, + stageScale: 1, + stageWidth: 100, + stageHeight: 100, + multiImage: false, + currentImage: 0, + checkLabels: jest.fn(() => true), + activeStates: jest.fn(() => []), + }; + const manager = { name: "image", obj: object, findSelectedTool: jest.fn(), selectTool: jest.fn() }; + const tool = Polygon.create({}, { manager, control, object }); + + return { tool, annotation, history, area }; +}; + +describe("Polygon freehand", () => { + beforeEach(() => { + jest.useFakeTimers(); + mockFreehandEnabled = false; + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + }); + + it("does not create a contour when the feature flag is off", () => { + const { tool, history } = createPolygonTool(); + + expect(tool.canStartFreehand()).toBe(false); + expect( + tool.commitFreehand([ + [10, 10], + [20, 10], + [20, 20], + ]), + ).toBe(false); + expect(history.freeze).not.toHaveBeenCalled(); + }); + + it("commits one contour inside a single history transaction", () => { + mockFreehandEnabled = true; + const { tool, annotation, history, area } = createPolygonTool(); + + expect( + tool.commitFreehand([ + [10, 10], + [20, 10], + [20, 20], + [10, 20], + ]), + ).toBe(true); + expect(history.freeze).toHaveBeenCalledTimes(1); + expect(history.freeze).toHaveBeenCalledWith("polygon-freehand"); + + jest.runOnlyPendingTimers(); + + expect(area.points).toHaveLength(4); + expect(area.closed).toBe(true); + expect(annotation.afterCreateResult).toHaveBeenCalledTimes(1); + expect(history.unfreeze).toHaveBeenCalledTimes(1); + expect(history.unfreeze).toHaveBeenCalledWith("polygon-freehand"); + }); + + it("releases history when deferred completion is skipped", () => { + mockFreehandEnabled = true; + const { tool, annotation, history, area } = createPolygonTool(); + + expect( + tool.commitFreehand([ + [10, 10], + [20, 10], + [20, 20], + ]), + ).toBe(true); + area.setDrawing(false); + annotation.setIsDrawing(false); + + jest.runOnlyPendingTimers(); + + expect(annotation.afterCreateResult).not.toHaveBeenCalled(); + expect(history.unfreeze).toHaveBeenCalledTimes(1); + expect(history.unfreeze).toHaveBeenCalledWith("polygon-freehand"); + }); + + it("releases history synchronously on a tool switch without double-finishing", () => { + mockFreehandEnabled = true; + const { tool, annotation, history } = createPolygonTool(); + + expect( + tool.commitFreehand([ + [10, 10], + [20, 10], + [20, 20], + ]), + ).toBe(true); + tool.handleToolSwitch({ toolName: "RectangleTool" }); + + expect(history.unfreeze).toHaveBeenCalledTimes(1); + jest.runOnlyPendingTimers(); + expect(annotation.afterCreateResult).toHaveBeenCalledTimes(1); + expect(history.unfreeze).toHaveBeenCalledTimes(1); + }); +}); diff --git a/web/libs/editor/src/utils/__tests__/freehand.test.js b/web/libs/editor/src/utils/__tests__/freehand.test.js new file mode 100644 index 000000000000..ded5f0dd8e78 --- /dev/null +++ b/web/libs/editor/src/utils/__tests__/freehand.test.js @@ -0,0 +1,52 @@ +import { appendFreehandPoint, simplifyFreehandPoints } from "../freehand"; + +describe("freehand utilities", () => { + it("keeps only endpoints for a straight trace", () => { + expect( + simplifyFreehandPoints( + [ + [0, 0], + [1, 0.1], + [2, -0.1], + [3, 0], + ], + 0.2, + ), + ).toEqual([ + { x: 0, y: 0 }, + { x: 3, y: 0 }, + ]); + }); + + it("preserves corners outside the Douglas-Peucker tolerance", () => { + expect( + simplifyFreehandPoints( + [ + [0, 0], + [5, 0], + [5, 5], + [10, 5], + ], + 1, + ), + ).toEqual([ + { x: 0, y: 0 }, + { x: 5, y: 0 }, + { x: 5, y: 5 }, + { x: 10, y: 5 }, + ]); + }); + + it("filters invalid, duplicate, and too-close samples", () => { + const first = appendFreehandPoint([], { x: 1, y: 1 }); + const unchanged = appendFreehandPoint(first, { x: 1.5, y: 1.5 }, 2); + const extended = appendFreehandPoint(unchanged, { x: 4, y: 1 }, 2); + + expect(unchanged).toBe(first); + expect(appendFreehandPoint(extended, { x: Number.NaN, y: 1 })).toBe(extended); + expect(extended).toEqual([ + { x: 1, y: 1 }, + { x: 4, y: 1 }, + ]); + }); +}); diff --git a/web/libs/editor/src/utils/feature-flags.ts b/web/libs/editor/src/utils/feature-flags.ts index 4b9ecc580a9a..99ffd3f046a8 100644 --- a/web/libs/editor/src/utils/feature-flags.ts +++ b/web/libs/editor/src/utils/feature-flags.ts @@ -144,6 +144,9 @@ export const FF_VIDEO_FRAME_SEEK_PRECISION = "fflag_fix_front_optic_1608_improve */ export const FF_FIT_1304_STRICT_OVERLAP = "fflag_feat_all_fit_1304_strict_overlap"; +/** Enable press-and-drag freehand drawing for Polygon tools. */ +export const FF_POLYGON_FREEHAND = "fflag_feat_front_polygon_freehand"; + Object.assign(window, { APP_SETTINGS: { ...(window.APP_SETTINGS ?? {}), diff --git a/web/libs/editor/src/utils/freehand.js b/web/libs/editor/src/utils/freehand.js new file mode 100644 index 000000000000..0cf13e2388ff --- /dev/null +++ b/web/libs/editor/src/utils/freehand.js @@ -0,0 +1,83 @@ +export const FREEHAND_MIN_DISTANCE = 2; +export const FREEHAND_SIMPLIFY_EPSILON = 2; + +const normalizePoint = (point) => { + const x = Array.isArray(point) ? point[0] : point?.x; + const y = Array.isArray(point) ? point[1] : point?.y; + + if (!Number.isFinite(x) || !Number.isFinite(y)) return null; + return Array.isArray(point) ? { x, y } : { ...point, x, y }; +}; + +const distanceSquared = (first, second) => { + const deltaX = first.x - second.x; + const deltaY = first.y - second.y; + + return deltaX * deltaX + deltaY * deltaY; +}; + +export function appendFreehandPoint(points, point, minDistance = FREEHAND_MIN_DISTANCE, force = false) { + const trace = Array.isArray(points) ? points : []; + const nextPoint = normalizePoint(point); + + if (!nextPoint) return trace; + if (trace.length === 0) return [nextPoint]; + + const pointDistanceSquared = distanceSquared(trace[trace.length - 1], nextPoint); + const threshold = Number.isFinite(minDistance) ? Math.max(0, minDistance) : FREEHAND_MIN_DISTANCE; + + if (pointDistanceSquared === 0 || (!force && pointDistanceSquared < threshold * threshold)) return trace; + return [...trace, nextPoint]; +} + +const pointToSegmentDistanceSquared = (point, start, end) => { + const segmentX = end.x - start.x; + const segmentY = end.y - start.y; + const segmentLengthSquared = segmentX * segmentX + segmentY * segmentY; + + if (segmentLengthSquared === 0) return distanceSquared(point, start); + + const projection = ((point.x - start.x) * segmentX + (point.y - start.y) * segmentY) / segmentLengthSquared; + const ratio = Math.max(0, Math.min(1, projection)); + return distanceSquared(point, { x: start.x + ratio * segmentX, y: start.y + ratio * segmentY }); +}; + +const markDouglasPeuckerPoints = (points, firstIndex, lastIndex, epsilonSquared, markers) => { + let furthestIndex = -1; + let furthestDistanceSquared = epsilonSquared; + + for (let index = firstIndex + 1; index < lastIndex; index++) { + const candidate = pointToSegmentDistanceSquared(points[index], points[firstIndex], points[lastIndex]); + + if (candidate > furthestDistanceSquared) { + furthestDistanceSquared = candidate; + furthestIndex = index; + } + } + + if (furthestIndex < 0) return; + markers[furthestIndex] = 1; + markDouglasPeuckerPoints(points, firstIndex, furthestIndex, epsilonSquared, markers); + markDouglasPeuckerPoints(points, furthestIndex, lastIndex, epsilonSquared, markers); +}; + +export function simplifyFreehandPoints(points, epsilon = FREEHAND_SIMPLIFY_EPSILON) { + const trace = (Array.isArray(points) ? points : []).reduce((result, point) => { + const normalized = normalizePoint(point); + + if (normalized && (!result.length || distanceSquared(result[result.length - 1], normalized) > 0)) { + result.push(normalized); + } + return result; + }, []); + + if (trace.length <= 2) return trace; + + const safeEpsilon = Number.isFinite(epsilon) ? Math.max(0, epsilon) : FREEHAND_SIMPLIFY_EPSILON; + const markers = new Uint8Array(trace.length); + const lastIndex = trace.length - 1; + + markers[0] = markers[lastIndex] = 1; + markDouglasPeuckerPoints(trace, 0, lastIndex, safeEpsilon * safeEpsilon, markers); + return trace.filter((_, index) => markers[index] === 1); +} diff --git a/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts b/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts new file mode 100644 index 000000000000..cc2d33a097bb --- /dev/null +++ b/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts @@ -0,0 +1,106 @@ +import { Hotkeys, ImageView, LabelStudio, Sidebar } from "@humansignal/frontend-test/helpers/LSF"; +import { FF_POLYGON_FREEHAND } from "../../../../src/utils/feature-flags"; +import { imageData, imageToolsConfig } from "../../data/image_segmentation/stage_interactions"; + +const drawFreehand = (pointerType = "mouse", includeCompatibilityMouseEvents = false) => { + ImageView.drawingArea.then(($element) => { + const { width, height } = $element[0].getBoundingClientRect(); + const points = [ + [width * 0.2, height * 0.2], + [width * 0.7, height * 0.2], + [width * 0.7, height * 0.7], + [width * 0.2, height * 0.7], + ]; + const pointer = { eventConstructor: "PointerEvent", pointerId: 1, pointerType, isPrimary: true }; + + let interaction = cy + .wrap($element) + .trigger("pointerdown", points[0][0], points[0][1], { ...pointer, button: 0, buttons: 1 }); + + if (includeCompatibilityMouseEvents) { + interaction = interaction.trigger("mousedown", points[0][0], points[0][1], { button: 0, buttons: 1 }); + } + + points.slice(1).forEach(([x, y]) => { + interaction = interaction.trigger("pointermove", x, y, { ...pointer, buttons: 1 }); + if (includeCompatibilityMouseEvents) interaction = interaction.trigger("mousemove", x, y, { buttons: 1 }); + }); + + interaction = interaction.trigger("pointerup", points[3][0], points[3][1], { + ...pointer, + button: 0, + buttons: 0, + }); + if (includeCompatibilityMouseEvents) { + interaction + .trigger("mouseup", points[3][0], points[3][1], { button: 0, buttons: 0 }) + .trigger("click", points[3][0], points[3][1], { button: 0, buttons: 0 }); + } + }); +}; + +const drawClickPolygon = () => { + ImageView.drawPolygonRelative( + [ + [0.2, 0.2], + [0.7, 0.2], + [0.7, 0.7], + [0.2, 0.7], + ], + true, + ); +}; + +describe("Freehand Polygon", () => { + it("ignores pointer drags while the feature flag is off", () => { + LabelStudio.addFeatureFlagsOnPageLoad({ [FF_POLYGON_FREEHAND]: false }); + LabelStudio.params().config(imageToolsConfig).data(imageData).withResult([]).init(); + LabelStudio.waitForImageReady(); + ImageView.selectPolygonToolByButton(); + + drawFreehand(); + + Sidebar.hasNoRegions(); + }); + + [false, true].forEach((freehandEnabled) => { + it(`preserves click-to-place drawing while the feature flag is ${freehandEnabled ? "on" : "off"}`, () => { + LabelStudio.addFeatureFlagsOnPageLoad({ [FF_POLYGON_FREEHAND]: freehandEnabled }); + LabelStudio.params().config(imageToolsConfig).data(imageData).withResult([]).init(); + LabelStudio.waitForImageReady(); + ImageView.selectPolygonToolByButton(); + + drawClickPolygon(); + + Sidebar.hasRegions(1); + }); + }); + + ["mouse", "pen", "touch"].forEach((pointerType) => { + it(`draws one undoable contour with ${pointerType} input`, () => { + LabelStudio.addFeatureFlagsOnPageLoad({ [FF_POLYGON_FREEHAND]: true }); + LabelStudio.params().config(imageToolsConfig).data(imageData).withResult([]).init(); + LabelStudio.waitForImageReady(); + ImageView.selectPolygonToolByButton(); + + drawFreehand(pointerType); + + Sidebar.hasRegions(1); + Hotkeys.undo(); + Sidebar.hasNoRegions(); + }); + }); + + it("creates only one contour when the browser emits compatibility mouse events", () => { + LabelStudio.addFeatureFlagsOnPageLoad({ [FF_POLYGON_FREEHAND]: true }); + LabelStudio.params().config(imageToolsConfig).data(imageData).withResult([]).init(); + LabelStudio.waitForImageReady(); + ImageView.selectPolygonToolByButton(); + + drawFreehand("mouse", true); + + Sidebar.hasRegions(1); + Hotkeys.undo(); + Sidebar.hasNoRegions(); + }); +});