From c4e480cbffcf8d9851fec10389869f5cb5439e3b Mon Sep 17 00:00:00 2001 From: ozymandiashh <234437643+ozymandiashh@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:22:55 +0300 Subject: [PATCH 1/3] feat: freehand polygon drawing (press-and-drag) behind feature flag Acceptance criteria: - When the flag is off, Polygon retains its existing click-to-place behavior and pointer drags create nothing. - When the flag is on, primary mouse, pen, and touch drags create one simplified polygon while ordinary clicks still place vertices. - Each sample keeps the coordinate transform active when it was collected, including during mid-drag zoom changes. - One undo action removes the entire completed contour. - Pointer cancellation, tool switches, teardown, and capture-less fallback leave no in-progress drawing or frozen history. - Unit and integration coverage exercises simplification, flag gating, input types, click coexistence, transform changes, and undo. Closes-PRD: #8315 References: HumanSignal/label-studio#8315 --- docs/source/tags/polygon.md | 4 + .../src/components/ImageView/ImageView.jsx | 260 +++++++++++++++++- .../components/ImageView/ImageView.module.css | 4 + .../ImageView/__tests__/ImageView.test.jsx | 119 +++++++- web/libs/editor/src/tools/Polygon.js | 98 ++++++- .../src/tools/__tests__/Polygon.test.js | 177 ++++++++++++ .../src/utils/__tests__/freehand.test.js | 52 ++++ web/libs/editor/src/utils/feature-flags.ts | 3 + web/libs/editor/src/utils/freehand.js | 83 ++++++ .../image_segmentation/freehand_polygon.cy.ts | 76 +++++ 10 files changed, 864 insertions(+), 12 deletions(-) create mode 100644 web/libs/editor/src/tools/__tests__/Polygon.test.js create mode 100644 web/libs/editor/src/utils/__tests__/freehand.test.js create mode 100644 web/libs/editor/src/utils/freehand.js create mode 100644 web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts diff --git a/docs/source/tags/polygon.md b/docs/source/tags/polygon.md index f64c9852c902..cd159dd62af0 100644 --- a/docs/source/tags/polygon.md +++ b/docs/source/tags/polygon.md @@ -12,6 +12,10 @@ Use with the following data types: image. {% insertmd includes/tags/polygon.md %} +### Freehand drawing + +When the `fflag_feat_front_polygon_freehand` feature flag is enabled, select the Polygon tool and press and drag to draw a contour with a mouse, pen, or primary touch. The contour is simplified when the pointer is released and is saved as one undoable polygon. The flag is off by default; with it disabled, the existing click-to-place behavior is unchanged. + ### Example Basic labeling configuration for polygonal image segmentation diff --git a/web/libs/editor/src/components/ImageView/ImageView.jsx b/web/libs/editor/src/components/ImageView/ImageView.jsx index 7e9be8ca7770..cb30ca2f9a94 100644 --- a/web/libs/editor/src/components/ImageView/ImageView.jsx +++ b/web/libs/editor/src/components/ImageView/ImageView.jsx @@ -20,14 +20,23 @@ import ResizeObserver from "../../utils/resize-observer"; import { debounce } from "@humansignal/core/lib/utils/debounce"; import Constants from "../../core/Constants"; import { fixRectToFit, mapKonvaBrightness } from "../../utils/image"; -import { FF_DEV_1442, FF_LSDV_4930, FF_ZOOM_OPTIM, isFF } from "../../utils/feature-flags"; +import { FF_DEV_1442, FF_LSDV_4930, FF_POLYGON_FREEHAND, FF_ZOOM_OPTIM, isFF } from "../../utils/feature-flags"; import { Pagination } from "../../common/Pagination/Pagination"; import { Image } from "./Image"; +import { + appendFreehandPoint, + FREEHAND_MIN_DISTANCE, + FREEHAND_SIMPLIFY_EPSILON, + simplifyFreehandPoints, +} from "../../utils/freehand"; Konva.showWarnings = false; const hotkeys = Hotkey("Image"); const imgDefaultProps = { crossOrigin: "anonymous" }; +const FREEHAND_COMPATIBILITY_GUARD_MS = 500; +const FREEHAND_COMPATIBILITY_GUARD_DISTANCE = 25; +const FREEHAND_DRAG_THRESHOLD = 5; export const splitRegions = (regions) => { const brushRegions = []; @@ -516,6 +525,7 @@ export default observer( state = { imgStyle: {}, pointer: [0, 0], + freehandPoints: [], }; imageRef = createRef(); @@ -527,6 +537,12 @@ export default observer( skipNextMouseUp = false; mouseDownPoint = null; mouseDown = false; + freehandPointerId = null; + freehandCaptureTarget = null; + freehandTool = null; + freehandTrace = []; + freehandDragging = false; + freehandCompatibilityGuard = null; constructor(props) { super(props); @@ -535,8 +551,212 @@ export default observer( props.store.settings.setSmoothing(props.item.smoothingEnabled); } + getFreehandTool = () => { + if (!isFF(FF_POLYGON_FREEHAND)) return null; + const tool = this.props.item.getToolsManager().findSelectedTool(); + + return tool?.toolName === "PolygonTool" && tool.canStartFreehand && tool.commitFreehand ? tool : null; + }; + + getFreehandPoint = (event) => { + const { item } = this.props; + const stage = item.stageRef; + + if (!stage) return null; + if (Number.isFinite(event?.clientX) && Number.isFinite(event?.clientY)) stage.setPointersPositions(event); + + const screen = stage.getPointerPosition(); + + if (!screen || !Number.isFinite(screen.x) || !Number.isFinite(screen.y)) return null; + + const [canvasX, canvasY] = item.fixZoomedCoords([screen.x, screen.y]); + + return { + screen, + canvas: { x: canvasX, y: canvasY }, + internal: [item.canvasToInternalX(canvasX), item.canvasToInternalY(canvasY)], + }; + }; + + appendFreehandEventPoint = (event, force = false) => { + const point = this.getFreehandPoint(event); + + if (!point) return false; + + const nextTrace = appendFreehandPoint( + this.freehandTrace, + { ...point.screen, internal: point.internal }, + FREEHAND_MIN_DISTANCE, + force, + ); + + if (nextTrace === this.freehandTrace) return false; + this.freehandTrace = nextTrace; + if (!this.freehandDragging) { + const first = this.freehandTrace[0]; + const last = this.freehandTrace[this.freehandTrace.length - 1]; + const deltaX = last.x - first.x; + const deltaY = last.y - first.y; + + this.freehandDragging = deltaX * deltaX + deltaY * deltaY >= FREEHAND_DRAG_THRESHOLD * FREEHAND_DRAG_THRESHOLD; + } + this.setState(({ freehandPoints }) => ({ + freehandPoints: [...freehandPoints, point.canvas.x, point.canvas.y], + })); + return true; + }; + + attachFreehandFallback = () => { + window.addEventListener("pointermove", this.handlePointerMove, true); + window.addEventListener("pointerup", this.handlePointerUp, true); + window.addEventListener("pointercancel", this.handlePointerCancel, true); + }; + + detachFreehandFallback = () => { + window.removeEventListener("pointermove", this.handlePointerMove, true); + window.removeEventListener("pointerup", this.handlePointerUp, true); + window.removeEventListener("pointercancel", this.handlePointerCancel, true); + }; + + resetFreehand = ({ updateState = true } = {}) => { + const pointerId = this.freehandPointerId; + const captureTarget = this.freehandCaptureTarget; + + this.freehandPointerId = null; + this.freehandCaptureTarget = null; + this.freehandTool = null; + this.freehandTrace = []; + this.freehandDragging = false; + this.detachFreehandFallback(); + captureTarget?.removeEventListener?.("lostpointercapture", this.handleLostPointerCapture); + if (pointerId !== null && captureTarget?.hasPointerCapture?.(pointerId)) { + try { + captureTarget.releasePointerCapture(pointerId); + } catch { + // Synthetic events and capture-less browsers may not own native capture. + } + } + + if (updateState) this.setState({ freehandPoints: [] }); + }; + + rememberFreehandCompatibilityEvent = (event) => { + const clientX = Number.isFinite(event?.clientX) ? event.clientX : event?.offsetX; + const clientY = Number.isFinite(event?.clientY) ? event.clientY : event?.offsetY; + + if (!Number.isFinite(clientX) || !Number.isFinite(clientY)) return; + this.freehandCompatibilityGuard = { + clientX, + clientY, + expiresAt: Date.now() + FREEHAND_COMPATIBILITY_GUARD_MS, + }; + }; + + shouldSuppressFreehandCompatibilityEvent = (event) => { + if (!isFF(FF_POLYGON_FREEHAND)) return false; + if (this.freehandDragging) return true; + + const guard = this.freehandCompatibilityGuard; + + if (!guard || Date.now() > guard.expiresAt) return false; + + const clientX = Number.isFinite(event?.clientX) ? event.clientX : event?.offsetX; + const clientY = Number.isFinite(event?.clientY) ? event.clientY : event?.offsetY; + const deltaX = clientX - guard.clientX; + const deltaY = clientY - guard.clientY; + + return ( + Number.isFinite(deltaX) && + Number.isFinite(deltaY) && + deltaX * deltaX + deltaY * deltaY <= + FREEHAND_COMPATIBILITY_GUARD_DISTANCE * FREEHAND_COMPATIBILITY_GUARD_DISTANCE + ); + }; + + handlePointerDown = (eventLike) => { + const event = eventLike.evt || eventLike; + + if (this.freehandPointerId !== null || event.isPrimary === false || (event.button ?? 0) !== 0) return; + + const tool = this.getFreehandTool(); + + if (!tool?.canStartFreehand()) return; + + const point = this.getFreehandPoint(event); + + if (!point) return; + + this.freehandCompatibilityGuard = null; + this.freehandPointerId = event.pointerId; + this.freehandTool = tool; + this.freehandTrace = appendFreehandPoint([], { ...point.screen, internal: point.internal }); + this.setState({ freehandPoints: [point.canvas.x, point.canvas.y] }); + + const captureTarget = this.props.item.stageRef?.content; + + this.freehandCaptureTarget = captureTarget ?? null; + captureTarget?.addEventListener?.("lostpointercapture", this.handleLostPointerCapture); + try { + captureTarget?.setPointerCapture?.(event.pointerId); + } catch { + // Continue with window listeners when pointer capture is unavailable. + } + + if (!captureTarget?.hasPointerCapture?.(event.pointerId)) this.attachFreehandFallback(); + }; + + handlePointerMove = (eventLike) => { + const event = eventLike.evt || eventLike; + + if (event.pointerId !== this.freehandPointerId) return; + + const coalescedSamples = event.getCoalescedEvents?.(); + const samples = coalescedSamples?.length ? coalescedSamples : [event]; + + for (const sample of samples) this.appendFreehandEventPoint(sample); + if (this.freehandDragging) event.preventDefault?.(); + }; + + handlePointerUp = (eventLike) => { + const event = eventLike.evt || eventLike; + + if (event.pointerId !== this.freehandPointerId) return; + + this.appendFreehandEventPoint(event, true); + const wasDragging = this.freehandDragging; + + if (wasDragging) { + event.preventDefault?.(); + this.rememberFreehandCompatibilityEvent(event); + const points = simplifyFreehandPoints(this.freehandTrace, FREEHAND_SIMPLIFY_EPSILON).map( + ({ internal }) => internal, + ); + + if (points.length >= 3 && this.freehandTool === this.getFreehandTool() && isAlive(this.freehandTool)) { + this.freehandTool.commitFreehand(points); + } + } + + this.resetFreehand(); + }; + + handlePointerCancel = (eventLike) => { + const event = eventLike.evt || eventLike; + + if (event.pointerId !== this.freehandPointerId) return; + if (this.freehandDragging) this.rememberFreehandCompatibilityEvent(event); + this.resetFreehand(); + }; + + handleLostPointerCapture = (event) => { + if (event.pointerId === this.freehandPointerId) this.resetFreehand(); + }; + handleOnClick = (e) => { const { item } = this.props; + const evt = e.evt || e; + + if (this.shouldSuppressFreehandCompatibilityEvent(evt)) return; if (isFF(FF_DEV_1442)) { this.handleDeferredMouseDown?.(true); @@ -546,7 +766,6 @@ export default observer( return; } - const evt = e.evt || e; const { offsetX: x, offsetY: y } = evt; if (isFF(FF_LSDV_4930)) { @@ -617,6 +836,7 @@ export default observer( }; handleMouseDown = (e) => { + if (this.shouldSuppressFreehandCompatibilityEvent(e.evt || e)) return true; this.mouseDown = true; const { item } = this.props; const isPanTool = item.getToolsManager().findSelectedTool()?.fullName === "ZoomPanTool"; @@ -753,6 +973,7 @@ export default observer( */ handleMouseUp = (e) => { this.mouseDown = false; + if (this.shouldSuppressFreehandCompatibilityEvent(e.evt || e)) return; const { item } = this.props; if (isFF(FF_DEV_1442)) { @@ -775,6 +996,7 @@ export default observer( }; handleMouseMove = (e) => { + if (this.shouldSuppressFreehandCompatibilityEvent(e.evt || e)) return; const { item } = this.props; item.freezeHistory(); @@ -984,6 +1206,7 @@ export default observer( }; componentWillUnmount() { + this.resetFreehand({ updateState: false }); this.detachObserver(); window.removeEventListener("resize", this.onResize); window.removeEventListener("mousemove", this.handleGlobalMouseMove); @@ -1152,6 +1375,10 @@ export default observer( onMouseDown={this.handleMouseDown} onMouseMove={this.handleMouseMove} onMouseUp={this.handleMouseUp} + onPointerDown={this.handlePointerDown} + onPointerMove={this.handlePointerMove} + onPointerUp={this.handlePointerUp} + onPointerCancel={this.handlePointerCancel} onWheel={item.zoom ? this.handleZoom : () => {}} /> ) : null} @@ -1191,6 +1418,10 @@ const EntireStage = observer( onMouseDown, onMouseMove, onMouseUp, + onPointerDown, + onPointerMove, + onPointerUp, + onPointerCancel, onWheel, crosshairRef, }) => { @@ -1220,7 +1451,13 @@ const EntireStage = observer( ref={(ref) => { item.setStageRef(ref); }} - className={[styles["image-element"], ...imagePositionClassnames].join(" ")} + className={[ + styles["image-element"], + ...imagePositionClassnames, + isFF(FF_POLYGON_FREEHAND) ? styles["freehand-enabled"] : null, + ] + .filter(Boolean) + .join(" ")} width={size.width} height={size.height} scaleX={item.zoomScale} @@ -1237,6 +1474,10 @@ const EntireStage = observer( onMouseDown={onMouseDown} onMouseMove={onMouseMove} onMouseUp={onMouseUp} + onPointerDown={onPointerDown} + onPointerMove={onPointerMove} + onPointerUp={onPointerUp} + onPointerCancel={onPointerCancel} onWheel={onWheel} > @@ -1440,6 +1681,19 @@ const StageContent = observer(({ item, store, state, crosshairRef }) => { ); })} + {state.freehandPoints.length >= 4 && ( + + + + )} {item.smoothingEnabled === false && } diff --git a/web/libs/editor/src/components/ImageView/ImageView.module.css b/web/libs/editor/src/components/ImageView/ImageView.module.css index 0fabef1a3319..2a644cb1bc3d 100644 --- a/web/libs/editor/src/components/ImageView/ImageView.module.css +++ b/web/libs/editor/src/components/ImageView/ImageView.module.css @@ -52,6 +52,10 @@ position: absolute; } +.freehand-enabled { + touch-action: none; +} + .image_position { position: absolute; diff --git a/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx b/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx index 63f942a61a27..a01a71a4a180 100644 --- a/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx +++ b/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx @@ -2,13 +2,14 @@ * Unit tests for ImageView (components/ImageView/ImageView.jsx) */ import React from "react"; -import { render, screen, fireEvent } from "@testing-library/react"; +import { act, render, screen, fireEvent } from "@testing-library/react"; import ImageView, { splitRegions } from "../ImageView"; jest.mock("../../../utils/feature-flags", () => ({ isFF: jest.fn(() => false), FF_DEV_1442: "fflag_dev_1442", FF_LSDV_4930: "fflag_lsdv_4930", + FF_POLYGON_FREEHAND: "fflag_feat_front_polygon_freehand", FF_ZOOM_OPTIM: "fflag_zoom_optim", })); @@ -207,6 +208,56 @@ function createStore(overrides = {}) { }; } +function createFreehandPointerHarness() { + const store = createStore(); + const commitFreehand = jest.fn(); + const tool = { + toolName: "PolygonTool", + canStartFreehand: jest.fn(() => true), + commitFreehand, + }; + let pointerPosition = { x: 10, y: 10 }; + let transformOffset = 0; + const captureTarget = { + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + setPointerCapture: jest.fn(), + hasPointerCapture: jest.fn(() => true), + releasePointerCapture: jest.fn(), + }; + const stageRef = { + content: captureTarget, + getPointerPosition: () => pointerPosition, + setPointersPositions: (event) => { + pointerPosition = { x: event.clientX, y: event.clientY }; + }, + on: jest.fn(), + off: jest.fn(), + getStage: () => null, + position: () => ({ x: 0, y: 0 }), + scale: () => ({ x: 1, y: 1 }), + }; + const item = createItem({ + stageRef, + getToolsManager: () => ({ findSelectedTool: () => tool, allTools: () => [tool] }), + fixZoomedCoords: ([x, y]) => [x + transformOffset, y + transformOffset], + canvasToInternalX: (x) => x, + canvasToInternalY: (y) => y, + }); + item.store = store; + let viewRef; + + render( (viewRef = ref)} item={item} store={store} />); + return { + commitFreehand, + item, + setTransformOffset: (offset) => { + transformOffset = offset; + }, + view: () => viewRef, + }; +} + describe("splitRegions", () => { it("returns empty arrays for empty regions", () => { const result = splitRegions([]); @@ -1222,4 +1273,70 @@ describe("ImageView with feature flags", () => { expect(item.event).toHaveBeenCalledWith("mousedown", expect.anything(), 20, 30); jest.useRealTimers(); }); + + it("creates points from a pointer drag only when freehand drawing is enabled", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, view } = createFreehandPointerHarness(); + const drag = () => { + act(() => { + view().handlePointerDown({ pointerId: 1, isPrimary: true, button: 0, clientX: 10, clientY: 10 }); + view().handlePointerMove({ + pointerId: 1, + clientX: 40, + clientY: 10, + getCoalescedEvents: () => [], + preventDefault: jest.fn(), + }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 10, clientY: 40, preventDefault: jest.fn() }); + }); + }; + + isFF.mockReturnValue(false); + drag(); + expect(commitFreehand).not.toHaveBeenCalled(); + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + drag(); + expect(commitFreehand).toHaveBeenCalledTimes(1); + expect(commitFreehand.mock.calls[0][0]).toHaveLength(4); + }); + + it("keeps each sample in the coordinate transform active when it was collected", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, setTransformOffset, view } = createFreehandPointerHarness(); + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + act(() => { + view().handlePointerDown({ pointerId: 1, isPrimary: true, button: 0, clientX: 10, clientY: 10 }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 10, preventDefault: jest.fn() }); + setTransformOffset(100); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 10, clientY: 40, preventDefault: jest.fn() }); + }); + + expect(commitFreehand).toHaveBeenCalledWith([ + [10, 10], + [40, 10], + [140, 140], + [110, 140], + ]); + }); + + it("lets a below-threshold pointer gesture continue through the stock click path", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, item, view } = createFreehandPointerHarness(); + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + act(() => { + view().handlePointerDown({ pointerId: 1, isPrimary: true, button: 0, clientX: 10, clientY: 10 }); + view().handlePointerMove({ pointerId: 1, clientX: 13, clientY: 10, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 13, clientY: 10, preventDefault: jest.fn() }); + }); + + expect(commitFreehand).not.toHaveBeenCalled(); + expect(view().shouldSuppressFreehandCompatibilityEvent({ clientX: 13, clientY: 10 })).toBe(false); + view().handleOnClick({ evt: { offsetX: 13, offsetY: 10, clientX: 13, clientY: 10 } }); + expect(item.event).toHaveBeenCalledWith("click", expect.anything(), 13, 10); + }); }); diff --git a/web/libs/editor/src/tools/Polygon.js b/web/libs/editor/src/tools/Polygon.js index b782909715da..df9faa211dab 100644 --- a/web/libs/editor/src/tools/Polygon.js +++ b/web/libs/editor/src/tools/Polygon.js @@ -6,6 +6,9 @@ import ToolMixin from "../mixins/Tool"; import { MultipleClicksDrawingTool } from "../mixins/DrawingTool"; import { NodeViews } from "../components/Node/Node"; import { observe } from "mobx"; +import { FF_POLYGON_FREEHAND, isFF } from "../utils/feature-flags"; + +const FREEHAND_HISTORY_KEY = "polygon-freehand"; const _Tool = types .model("PolygonTool", { @@ -75,10 +78,76 @@ const _Tool = types .actions((self) => { let disposer; let closed; + let freehandHistoryFrozen = false; + let freehandFinishTimer = null; + + const releaseFreehandHistory = () => { + if (!freehandHistoryFrozen) return; + freehandHistoryFrozen = false; + self.annotation.history.unfreeze(FREEHAND_HISTORY_KEY); + }; + + const finishFreehandDrawing = () => { + if (!self.isDrawing || !self.getCurrentArea()) { + releaseFreehandHistory(); + return false; + } + + self.annotation.regionStore.selection.drawingUnselect(); + self.closeCurrent(); + freehandFinishTimer = setTimeout(() => { + freehandFinishTimer = null; + if (!isAlive(self) || !self.isDrawing || !self.annotation.isDrawing || !self.getCurrentArea()) { + releaseFreehandHistory(); + return; + } + self._finishDrawing(); + }); + return true; + }; return { + canStartFreehand() { + return isFF(FF_POLYGON_FREEHAND) && self.canStartDrawing(); + }, + commitFreehand(points) { + if (!Array.isArray(points) || points.length < 3 || !self.canStartFreehand()) return false; + + const validPoints = points.filter( + (point) => Array.isArray(point) && Number.isFinite(point[0]) && Number.isFinite(point[1]), + ); + + if (validPoints.length < 3) return false; + + self.stopListening(); + closed = false; + self.annotation.history.freeze(FREEHAND_HISTORY_KEY); + freehandHistoryFrozen = true; + try { + self.startDrawing(validPoints[0][0], validPoints[0][1]); + + if (!self.isDrawing || !self.getCurrentArea()) { + releaseFreehandHistory(); + return false; + } + + validPoints.slice(1).forEach(([x, y]) => self.nextPoint(x, y)); + if (self.getCurrentArea().points.length < 3) { + self.cleanupUncloseableShape(); + releaseFreehandHistory(); + return false; + } + + return finishFreehandDrawing(); + } catch (error) { + releaseFreehandHistory(); + throw error; + } + }, handleToolSwitch(tool) { self.stopListening(); + releaseFreehandHistory(); + if (freehandFinishTimer !== null) return; if (self.getCurrentArea()?.isDrawing && tool.toolName !== "ZoomPanTool") { const shape = self.getCurrentArea()?.toJSON(); @@ -100,7 +169,16 @@ const _Tool = types ); }, stopListening() { - if (disposer) disposer(); + if (disposer) { + disposer(); + disposer = null; + } + }, + beforeDestroy() { + self.stopListening(); + if (freehandFinishTimer !== null) clearTimeout(freehandFinishTimer); + freehandFinishTimer = null; + releaseFreehandHistory(); }, closeCurrent() { self.stopListening(); @@ -121,13 +199,17 @@ const _Tool = types }, _finishDrawing() { - const { currentArea, control } = self; - - self.currentArea.notifyDrawingFinished(); - self.setDrawing(false); - self.currentArea = null; - self.mode = "viewing"; - self.annotation.afterCreateResult(currentArea, control); + try { + const { currentArea, control } = self; + + self.currentArea.notifyDrawingFinished(); + self.setDrawing(false); + self.currentArea = null; + self.mode = "viewing"; + self.annotation.afterCreateResult(currentArea, control); + } finally { + releaseFreehandHistory(); + } }, setDrawing(drawing) { diff --git a/web/libs/editor/src/tools/__tests__/Polygon.test.js b/web/libs/editor/src/tools/__tests__/Polygon.test.js new file mode 100644 index 000000000000..17e414fb40ea --- /dev/null +++ b/web/libs/editor/src/tools/__tests__/Polygon.test.js @@ -0,0 +1,177 @@ +let mockFreehandEnabled = false; + +jest.mock("../../utils/feature-flags", () => ({ + FF_DEV_3391: "fflag_fix_front_dev_3391_interactive_view_all", + FF_SIMPLE_INIT: "fflag_fix_front_leap_443_select_annotation_once", + FF_POLYGON_FREEHAND: "fflag_feat_front_polygon_freehand", + isFF: jest.fn((flag) => { + if (flag === "fflag_feat_front_polygon_freehand") return mockFreehandEnabled; + return flag === "fflag_fix_front_dev_3391_interactive_view_all"; + }), +})); + +jest.mock("@humansignal/core", () => ({ + ff: { + FF_MULTIPLE_LABELS_REGIONS: "fflag_multiple_labels_regions", + isActive: jest.fn(() => false), + }, +})); + +jest.mock("../../components/Node/Node", () => ({ + NodeViews: { PolygonRegionModel: { icon: jest.fn(), altIcon: jest.fn() } }, +})); + +const { Polygon } = require("../Polygon"); + +const createPolygonTool = () => { + const history = { freeze: jest.fn(), unfreeze: jest.fn() }; + const selection = { drawingUnselect: jest.fn(), hasSelection: false }; + const annotation = { + editable: true, + isDrawing: false, + isReadOnly: jest.fn(() => false), + history, + regionStore: { selection, hasSelection: false }, + setIsDrawing: jest.fn((drawing) => { + annotation.isDrawing = drawing; + }), + afterCreateResult: jest.fn(), + unselectAll: jest.fn(), + }; + const control = { + type: "polygonlabels", + isSelected: true, + isSeparated: false, + annotation, + getResultValue: jest.fn(() => ({})), + getSnappedPoint: jest.fn(({ x, y }) => ({ x, y })), + }; + const area = { + type: "polygonregion", + closed: false, + isDrawing: true, + points: [], + setValue: jest.fn(), + setDrawing: jest.fn((drawing) => { + area.isDrawing = drawing; + }), + addPoint: jest.fn((x, y) => area.points.push({ x, y })), + closePoly: jest.fn(() => { + area.closed = true; + }), + notifyDrawingFinished: jest.fn(), + toJSON: jest.fn(() => ({ points: area.points.map(({ x, y }) => [x, y]) })), + }; + annotation.createResult = jest.fn((options) => { + area.points = options.points.map(([x, y]) => ({ x, y })); + return area; + }); + const object = { + name: "image", + annotation, + regs: [], + canvasSize: { width: 100, height: 100 }, + stageScale: 1, + stageWidth: 100, + stageHeight: 100, + multiImage: false, + currentImage: 0, + checkLabels: jest.fn(() => true), + activeStates: jest.fn(() => []), + }; + const manager = { name: "image", obj: object, findSelectedTool: jest.fn(), selectTool: jest.fn() }; + const tool = Polygon.create({}, { manager, control, object }); + + return { tool, annotation, history, area }; +}; + +describe("Polygon freehand", () => { + beforeEach(() => { + jest.useFakeTimers(); + mockFreehandEnabled = false; + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.runOnlyPendingTimers(); + jest.useRealTimers(); + }); + + it("does not create a contour when the feature flag is off", () => { + const { tool, history } = createPolygonTool(); + + expect(tool.canStartFreehand()).toBe(false); + expect( + tool.commitFreehand([ + [10, 10], + [20, 10], + [20, 20], + ]), + ).toBe(false); + expect(history.freeze).not.toHaveBeenCalled(); + }); + + it("commits one contour inside a single history transaction", () => { + mockFreehandEnabled = true; + const { tool, annotation, history, area } = createPolygonTool(); + + expect( + tool.commitFreehand([ + [10, 10], + [20, 10], + [20, 20], + [10, 20], + ]), + ).toBe(true); + expect(history.freeze).toHaveBeenCalledTimes(1); + expect(history.freeze).toHaveBeenCalledWith("polygon-freehand"); + + jest.runOnlyPendingTimers(); + + expect(area.points).toHaveLength(4); + expect(area.closed).toBe(true); + expect(annotation.afterCreateResult).toHaveBeenCalledTimes(1); + expect(history.unfreeze).toHaveBeenCalledTimes(1); + expect(history.unfreeze).toHaveBeenCalledWith("polygon-freehand"); + }); + + it("releases history when deferred completion is skipped", () => { + mockFreehandEnabled = true; + const { tool, annotation, history, area } = createPolygonTool(); + + expect( + tool.commitFreehand([ + [10, 10], + [20, 10], + [20, 20], + ]), + ).toBe(true); + area.setDrawing(false); + annotation.setIsDrawing(false); + + jest.runOnlyPendingTimers(); + + expect(annotation.afterCreateResult).not.toHaveBeenCalled(); + expect(history.unfreeze).toHaveBeenCalledTimes(1); + expect(history.unfreeze).toHaveBeenCalledWith("polygon-freehand"); + }); + + it("releases history synchronously on a tool switch without double-finishing", () => { + mockFreehandEnabled = true; + const { tool, annotation, history } = createPolygonTool(); + + expect( + tool.commitFreehand([ + [10, 10], + [20, 10], + [20, 20], + ]), + ).toBe(true); + tool.handleToolSwitch({ toolName: "RectangleTool" }); + + expect(history.unfreeze).toHaveBeenCalledTimes(1); + jest.runOnlyPendingTimers(); + expect(annotation.afterCreateResult).toHaveBeenCalledTimes(1); + expect(history.unfreeze).toHaveBeenCalledTimes(1); + }); +}); diff --git a/web/libs/editor/src/utils/__tests__/freehand.test.js b/web/libs/editor/src/utils/__tests__/freehand.test.js new file mode 100644 index 000000000000..ded5f0dd8e78 --- /dev/null +++ b/web/libs/editor/src/utils/__tests__/freehand.test.js @@ -0,0 +1,52 @@ +import { appendFreehandPoint, simplifyFreehandPoints } from "../freehand"; + +describe("freehand utilities", () => { + it("keeps only endpoints for a straight trace", () => { + expect( + simplifyFreehandPoints( + [ + [0, 0], + [1, 0.1], + [2, -0.1], + [3, 0], + ], + 0.2, + ), + ).toEqual([ + { x: 0, y: 0 }, + { x: 3, y: 0 }, + ]); + }); + + it("preserves corners outside the Douglas-Peucker tolerance", () => { + expect( + simplifyFreehandPoints( + [ + [0, 0], + [5, 0], + [5, 5], + [10, 5], + ], + 1, + ), + ).toEqual([ + { x: 0, y: 0 }, + { x: 5, y: 0 }, + { x: 5, y: 5 }, + { x: 10, y: 5 }, + ]); + }); + + it("filters invalid, duplicate, and too-close samples", () => { + const first = appendFreehandPoint([], { x: 1, y: 1 }); + const unchanged = appendFreehandPoint(first, { x: 1.5, y: 1.5 }, 2); + const extended = appendFreehandPoint(unchanged, { x: 4, y: 1 }, 2); + + expect(unchanged).toBe(first); + expect(appendFreehandPoint(extended, { x: Number.NaN, y: 1 })).toBe(extended); + expect(extended).toEqual([ + { x: 1, y: 1 }, + { x: 4, y: 1 }, + ]); + }); +}); diff --git a/web/libs/editor/src/utils/feature-flags.ts b/web/libs/editor/src/utils/feature-flags.ts index 4b9ecc580a9a..99ffd3f046a8 100644 --- a/web/libs/editor/src/utils/feature-flags.ts +++ b/web/libs/editor/src/utils/feature-flags.ts @@ -144,6 +144,9 @@ export const FF_VIDEO_FRAME_SEEK_PRECISION = "fflag_fix_front_optic_1608_improve */ export const FF_FIT_1304_STRICT_OVERLAP = "fflag_feat_all_fit_1304_strict_overlap"; +/** Enable press-and-drag freehand drawing for Polygon tools. */ +export const FF_POLYGON_FREEHAND = "fflag_feat_front_polygon_freehand"; + Object.assign(window, { APP_SETTINGS: { ...(window.APP_SETTINGS ?? {}), diff --git a/web/libs/editor/src/utils/freehand.js b/web/libs/editor/src/utils/freehand.js new file mode 100644 index 000000000000..0cf13e2388ff --- /dev/null +++ b/web/libs/editor/src/utils/freehand.js @@ -0,0 +1,83 @@ +export const FREEHAND_MIN_DISTANCE = 2; +export const FREEHAND_SIMPLIFY_EPSILON = 2; + +const normalizePoint = (point) => { + const x = Array.isArray(point) ? point[0] : point?.x; + const y = Array.isArray(point) ? point[1] : point?.y; + + if (!Number.isFinite(x) || !Number.isFinite(y)) return null; + return Array.isArray(point) ? { x, y } : { ...point, x, y }; +}; + +const distanceSquared = (first, second) => { + const deltaX = first.x - second.x; + const deltaY = first.y - second.y; + + return deltaX * deltaX + deltaY * deltaY; +}; + +export function appendFreehandPoint(points, point, minDistance = FREEHAND_MIN_DISTANCE, force = false) { + const trace = Array.isArray(points) ? points : []; + const nextPoint = normalizePoint(point); + + if (!nextPoint) return trace; + if (trace.length === 0) return [nextPoint]; + + const pointDistanceSquared = distanceSquared(trace[trace.length - 1], nextPoint); + const threshold = Number.isFinite(minDistance) ? Math.max(0, minDistance) : FREEHAND_MIN_DISTANCE; + + if (pointDistanceSquared === 0 || (!force && pointDistanceSquared < threshold * threshold)) return trace; + return [...trace, nextPoint]; +} + +const pointToSegmentDistanceSquared = (point, start, end) => { + const segmentX = end.x - start.x; + const segmentY = end.y - start.y; + const segmentLengthSquared = segmentX * segmentX + segmentY * segmentY; + + if (segmentLengthSquared === 0) return distanceSquared(point, start); + + const projection = ((point.x - start.x) * segmentX + (point.y - start.y) * segmentY) / segmentLengthSquared; + const ratio = Math.max(0, Math.min(1, projection)); + return distanceSquared(point, { x: start.x + ratio * segmentX, y: start.y + ratio * segmentY }); +}; + +const markDouglasPeuckerPoints = (points, firstIndex, lastIndex, epsilonSquared, markers) => { + let furthestIndex = -1; + let furthestDistanceSquared = epsilonSquared; + + for (let index = firstIndex + 1; index < lastIndex; index++) { + const candidate = pointToSegmentDistanceSquared(points[index], points[firstIndex], points[lastIndex]); + + if (candidate > furthestDistanceSquared) { + furthestDistanceSquared = candidate; + furthestIndex = index; + } + } + + if (furthestIndex < 0) return; + markers[furthestIndex] = 1; + markDouglasPeuckerPoints(points, firstIndex, furthestIndex, epsilonSquared, markers); + markDouglasPeuckerPoints(points, furthestIndex, lastIndex, epsilonSquared, markers); +}; + +export function simplifyFreehandPoints(points, epsilon = FREEHAND_SIMPLIFY_EPSILON) { + const trace = (Array.isArray(points) ? points : []).reduce((result, point) => { + const normalized = normalizePoint(point); + + if (normalized && (!result.length || distanceSquared(result[result.length - 1], normalized) > 0)) { + result.push(normalized); + } + return result; + }, []); + + if (trace.length <= 2) return trace; + + const safeEpsilon = Number.isFinite(epsilon) ? Math.max(0, epsilon) : FREEHAND_SIMPLIFY_EPSILON; + const markers = new Uint8Array(trace.length); + const lastIndex = trace.length - 1; + + markers[0] = markers[lastIndex] = 1; + markDouglasPeuckerPoints(trace, 0, lastIndex, safeEpsilon * safeEpsilon, markers); + return trace.filter((_, index) => markers[index] === 1); +} diff --git a/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts b/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts new file mode 100644 index 000000000000..7b42113eba60 --- /dev/null +++ b/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts @@ -0,0 +1,76 @@ +import { Hotkeys, ImageView, LabelStudio, Sidebar } from "@humansignal/frontend-test/helpers/LSF"; +import { FF_POLYGON_FREEHAND } from "../../../../src/utils/feature-flags"; +import { imageData, imageToolsConfig } from "../../data/image_segmentation/stage_interactions"; + +const drawFreehand = (pointerType = "mouse") => { + ImageView.drawingArea.then(($element) => { + const { width, height } = $element[0].getBoundingClientRect(); + const points = [ + [width * 0.2, height * 0.2], + [width * 0.7, height * 0.2], + [width * 0.7, height * 0.7], + [width * 0.2, height * 0.7], + ]; + const pointer = { eventConstructor: "PointerEvent", pointerId: 1, pointerType, isPrimary: true }; + + cy.wrap($element) + .trigger("pointerdown", points[0][0], points[0][1], { ...pointer, button: 0, buttons: 1 }) + .trigger("pointermove", points[1][0], points[1][1], { ...pointer, buttons: 1 }) + .trigger("pointermove", points[2][0], points[2][1], { ...pointer, buttons: 1 }) + .trigger("pointermove", points[3][0], points[3][1], { ...pointer, buttons: 1 }) + .trigger("pointerup", points[3][0], points[3][1], { ...pointer, button: 0, buttons: 0 }); + }); +}; + +const drawClickPolygon = () => { + ImageView.drawPolygonRelative( + [ + [0.2, 0.2], + [0.7, 0.2], + [0.7, 0.7], + [0.2, 0.7], + ], + true, + ); +}; + +describe("Freehand Polygon", () => { + it("ignores pointer drags while the feature flag is off", () => { + LabelStudio.addFeatureFlagsOnPageLoad({ [FF_POLYGON_FREEHAND]: false }); + LabelStudio.params().config(imageToolsConfig).data(imageData).withResult([]).init(); + LabelStudio.waitForImageReady(); + ImageView.selectPolygonToolByButton(); + + drawFreehand(); + + Sidebar.hasNoRegions(); + }); + + [false, true].forEach((freehandEnabled) => { + it(`preserves click-to-place drawing while the feature flag is ${freehandEnabled ? "on" : "off"}`, () => { + LabelStudio.addFeatureFlagsOnPageLoad({ [FF_POLYGON_FREEHAND]: freehandEnabled }); + LabelStudio.params().config(imageToolsConfig).data(imageData).withResult([]).init(); + LabelStudio.waitForImageReady(); + ImageView.selectPolygonToolByButton(); + + drawClickPolygon(); + + Sidebar.hasRegions(1); + }); + }); + + ["mouse", "pen", "touch"].forEach((pointerType) => { + it(`draws one undoable contour with ${pointerType} input`, () => { + LabelStudio.addFeatureFlagsOnPageLoad({ [FF_POLYGON_FREEHAND]: true }); + LabelStudio.params().config(imageToolsConfig).data(imageData).withResult([]).init(); + LabelStudio.waitForImageReady(); + ImageView.selectPolygonToolByButton(); + + drawFreehand(pointerType); + + Sidebar.hasRegions(1); + Hotkeys.undo(); + Sidebar.hasNoRegions(); + }); + }); +}); From 4cb16c785f0cb8c37500f59585193e59da4601ee Mon Sep 17 00:00:00 2001 From: ozymandiashh <234437643+ozymandiashh@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:57:35 +0300 Subject: [PATCH 2/3] fix: preserve freehand polygon interaction targets --- .../src/components/ImageView/ImageView.jsx | 69 +++++---- .../ImageView/__tests__/ImageView.test.jsx | 141 +++++++++++++++++- .../image_segmentation/freehand_polygon.cy.ts | 44 +++++- 3 files changed, 212 insertions(+), 42 deletions(-) diff --git a/web/libs/editor/src/components/ImageView/ImageView.jsx b/web/libs/editor/src/components/ImageView/ImageView.jsx index cb30ca2f9a94..4f93cecad41e 100644 --- a/web/libs/editor/src/components/ImageView/ImageView.jsx +++ b/web/libs/editor/src/components/ImageView/ImageView.jsx @@ -558,6 +558,35 @@ export default observer( return tool?.toolName === "PolygonTool" && tool.canStartFreehand && tool.commitFreehand ? tool : null; }; + isRightElementToCatchToolInteractions = (element, isMoveTool) => { + // Bitmask is like Brush, so treat it the same. The only difference is + // that Bitmask doesn't have a group inside. + if (element.nodeType === "Layer" && !isMoveTool && element.attrs?.name === "bitmask") return true; + + if (element.nodeType === "Group") { + // It could be ruler or segmentation. + if (element.attrs?.name === "ruler") return true; + + // Segmentation is specific for Brushes, but click interaction on the + // region covers the same MoveTool interaction, so ignore MoveTool here + // to prevent conflicts. + if (!isMoveTool && element.attrs?.name === "segmentation") return true; + } + + return false; + }; + + isToolInteractionTarget = (target, isMoveTool = false) => { + const { item } = this.props; + + if (!target) return false; + return ( + item.getSkipInteractions() || + target === item.stageRef || + findClosestParent(target, (element) => this.isRightElementToCatchToolInteractions(element, isMoveTool)) + ); + }; + getFreehandPoint = (event) => { const { item } = this.props; const stage = item.stageRef; @@ -682,6 +711,13 @@ export default observer( if (!tool?.canStartFreehand()) return; + const { item } = this.props; + + item.updateSkipInteractions(eventLike); + if (item.annotation.isReadOnly()) return; + if (eventLike.target?.getParent?.()?.className === "Transformer") return; + if (!this.isToolInteractionTarget(eventLike.target)) return; + const point = this.getFreehandPoint(event); if (!point) return; @@ -860,34 +896,7 @@ export default observer( e.evt.preventDefault(); } - const isRightElementToCatchToolInteractions = (el) => { - // Bitmask is like Brush, so treat it the same - // The only difference is that Bitmask doesn't have a group inside - if (el.nodeType === "Layer" && !isMoveTool && el.attrs?.name === "bitmask") { - return true; - } - - // It could be ruler ot segmentation - if (el.nodeType === "Group") { - if (el?.attrs?.name === "ruler") { - return true; - } - // segmentation is specific for Brushes - // but click interaction on the region covers the case of the same MoveTool interaction here, - // so it should ignore move tool interaction to prevent conflicts - if (!isMoveTool && el?.attrs?.name === "segmentation") { - return true; - } - } - return false; - }; - - if ( - // create regions over another regions with Cmd/Ctrl pressed - item.getSkipInteractions() || - e.target === item.stageRef || - findClosestParent(e.target, isRightElementToCatchToolInteractions) - ) { + if (this.isToolInteractionTarget(e.target, isMoveTool)) { window.addEventListener("mousemove", this.handleGlobalMouseMove); window.addEventListener("mouseup", this.handleGlobalMouseUp); const { offsetX: x, offsetY: y } = e.evt; @@ -1426,6 +1435,8 @@ const EntireStage = observer( crosshairRef, }) => { const { store } = item; + const selectedTool = item.getToolsManager().findSelectedTool(); + const freehandEnabled = isFF(FF_POLYGON_FREEHAND) && selectedTool?.toolName === "PolygonTool"; let size; let position; @@ -1454,7 +1465,7 @@ const EntireStage = observer( className={[ styles["image-element"], ...imagePositionClassnames, - isFF(FF_POLYGON_FREEHAND) ? styles["freehand-enabled"] : null, + freehandEnabled ? styles["freehand-enabled"] : null, ] .filter(Boolean) .join(" ")} diff --git a/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx b/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx index a01a71a4a180..f2794bbc29fd 100644 --- a/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx +++ b/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx @@ -251,6 +251,7 @@ function createFreehandPointerHarness() { return { commitFreehand, item, + stageRef, setTransformOffset: (offset) => { transformOffset = offset; }, @@ -1276,10 +1277,17 @@ describe("ImageView with feature flags", () => { it("creates points from a pointer drag only when freehand drawing is enabled", () => { const { isFF } = require("../../../utils/feature-flags"); - const { commitFreehand, view } = createFreehandPointerHarness(); + const { commitFreehand, stageRef, view } = createFreehandPointerHarness(); const drag = () => { act(() => { - view().handlePointerDown({ pointerId: 1, isPrimary: true, button: 0, clientX: 10, clientY: 10 }); + view().handlePointerDown({ + target: stageRef, + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); view().handlePointerMove({ pointerId: 1, clientX: 40, @@ -1304,11 +1312,18 @@ describe("ImageView with feature flags", () => { it("keeps each sample in the coordinate transform active when it was collected", () => { const { isFF } = require("../../../utils/feature-flags"); - const { commitFreehand, setTransformOffset, view } = createFreehandPointerHarness(); + const { commitFreehand, setTransformOffset, stageRef, view } = createFreehandPointerHarness(); isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); act(() => { - view().handlePointerDown({ pointerId: 1, isPrimary: true, button: 0, clientX: 10, clientY: 10 }); + view().handlePointerDown({ + target: stageRef, + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 10, preventDefault: jest.fn() }); setTransformOffset(100); view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 40, preventDefault: jest.fn() }); @@ -1325,11 +1340,18 @@ describe("ImageView with feature flags", () => { it("lets a below-threshold pointer gesture continue through the stock click path", () => { const { isFF } = require("../../../utils/feature-flags"); - const { commitFreehand, item, view } = createFreehandPointerHarness(); + const { commitFreehand, item, stageRef, view } = createFreehandPointerHarness(); isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); act(() => { - view().handlePointerDown({ pointerId: 1, isPrimary: true, button: 0, clientX: 10, clientY: 10 }); + view().handlePointerDown({ + target: stageRef, + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); view().handlePointerMove({ pointerId: 1, clientX: 13, clientY: 10, preventDefault: jest.fn() }); view().handlePointerUp({ pointerId: 1, clientX: 13, clientY: 10, preventDefault: jest.fn() }); }); @@ -1339,4 +1361,111 @@ describe("ImageView with feature flags", () => { view().handleOnClick({ evt: { offsetX: 13, offsetY: 10, clientX: 13, clientY: 10 } }); expect(item.event).toHaveBeenCalledWith("click", expect.anything(), 13, 10); }); + + it("does not start freehand drawing from an existing region or transformer", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, view } = createFreehandPointerHarness(); + const dragFrom = (target, pointerId) => { + act(() => { + view().handlePointerDown({ + target, + pointerId, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); + view().handlePointerMove({ pointerId, clientX: 40, clientY: 10, preventDefault: jest.fn() }); + view().handlePointerMove({ pointerId, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId, clientX: 10, clientY: 40, preventDefault: jest.fn() }); + }); + }; + const region = { parent: null, getParent: () => null }; + const transformer = { className: "Transformer" }; + const anchor = { parent: transformer, getParent: () => transformer }; + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + dragFrom(region, 1); + dragFrom(anchor, 2); + + expect(commitFreehand).not.toHaveBeenCalled(); + expect(view().freehandPointerId).toBeNull(); + }); + + it("does not start freehand drawing on a read-only annotation", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, item, stageRef, view } = createFreehandPointerHarness(); + + item.annotation.isReadOnly = () => true; + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + act(() => { + view().handlePointerDown({ + target: stageRef, + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 10, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + }); + + expect(commitFreehand).not.toHaveBeenCalled(); + expect(view().freehandPointerId).toBeNull(); + }); + + it("allows freehand drawing over a region when Cmd/Ctrl interaction skipping is active", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { commitFreehand, item, view } = createFreehandPointerHarness(); + const region = { parent: null, getParent: () => null }; + let skipInteractions = false; + + item.updateSkipInteractions = jest.fn(({ evt }) => { + skipInteractions = evt.metaKey || evt.ctrlKey; + }); + item.getSkipInteractions = () => skipInteractions; + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + act(() => { + view().handlePointerDown({ + target: region, + evt: { + pointerId: 1, + isPrimary: true, + button: 0, + clientX: 10, + clientY: 10, + metaKey: true, + }, + }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 10, preventDefault: jest.fn() }); + view().handlePointerMove({ pointerId: 1, clientX: 40, clientY: 40, preventDefault: jest.fn() }); + view().handlePointerUp({ pointerId: 1, clientX: 10, clientY: 40, preventDefault: jest.fn() }); + }); + + expect(item.updateSkipInteractions).toHaveBeenCalledTimes(1); + expect(commitFreehand).toHaveBeenCalledTimes(1); + }); + + it("enables touch-action suppression only while Polygon is selected", () => { + const { isFF } = require("../../../utils/feature-flags"); + const store = createStore(); + const polygonTool = { toolName: "PolygonTool" }; + const polygonItem = createItem({ + getToolsManager: () => ({ findSelectedTool: () => polygonTool, allTools: () => [polygonTool] }), + }); + const rectangleTool = { toolName: "RectangleTool" }; + const rectangleItem = createItem({ + getToolsManager: () => ({ findSelectedTool: () => rectangleTool, allTools: () => [rectangleTool] }), + }); + + polygonItem.store = store; + rectangleItem.store = store; + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + const { rerender } = render(); + expect(screen.getByTestId("konva-stage").className).toContain("freehand-enabled"); + + rerender(); + expect(screen.getByTestId("konva-stage").className).not.toContain("freehand-enabled"); + }); }); diff --git a/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts b/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts index 7b42113eba60..cc2d33a097bb 100644 --- a/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts +++ b/web/libs/editor/tests/integration/e2e/image_segmentation/freehand_polygon.cy.ts @@ -2,7 +2,7 @@ import { Hotkeys, ImageView, LabelStudio, Sidebar } from "@humansignal/frontend- import { FF_POLYGON_FREEHAND } from "../../../../src/utils/feature-flags"; import { imageData, imageToolsConfig } from "../../data/image_segmentation/stage_interactions"; -const drawFreehand = (pointerType = "mouse") => { +const drawFreehand = (pointerType = "mouse", includeCompatibilityMouseEvents = false) => { ImageView.drawingArea.then(($element) => { const { width, height } = $element[0].getBoundingClientRect(); const points = [ @@ -13,12 +13,29 @@ const drawFreehand = (pointerType = "mouse") => { ]; const pointer = { eventConstructor: "PointerEvent", pointerId: 1, pointerType, isPrimary: true }; - cy.wrap($element) - .trigger("pointerdown", points[0][0], points[0][1], { ...pointer, button: 0, buttons: 1 }) - .trigger("pointermove", points[1][0], points[1][1], { ...pointer, buttons: 1 }) - .trigger("pointermove", points[2][0], points[2][1], { ...pointer, buttons: 1 }) - .trigger("pointermove", points[3][0], points[3][1], { ...pointer, buttons: 1 }) - .trigger("pointerup", points[3][0], points[3][1], { ...pointer, button: 0, buttons: 0 }); + let interaction = cy + .wrap($element) + .trigger("pointerdown", points[0][0], points[0][1], { ...pointer, button: 0, buttons: 1 }); + + if (includeCompatibilityMouseEvents) { + interaction = interaction.trigger("mousedown", points[0][0], points[0][1], { button: 0, buttons: 1 }); + } + + points.slice(1).forEach(([x, y]) => { + interaction = interaction.trigger("pointermove", x, y, { ...pointer, buttons: 1 }); + if (includeCompatibilityMouseEvents) interaction = interaction.trigger("mousemove", x, y, { buttons: 1 }); + }); + + interaction = interaction.trigger("pointerup", points[3][0], points[3][1], { + ...pointer, + button: 0, + buttons: 0, + }); + if (includeCompatibilityMouseEvents) { + interaction + .trigger("mouseup", points[3][0], points[3][1], { button: 0, buttons: 0 }) + .trigger("click", points[3][0], points[3][1], { button: 0, buttons: 0 }); + } }); }; @@ -73,4 +90,17 @@ describe("Freehand Polygon", () => { Sidebar.hasNoRegions(); }); }); + + it("creates only one contour when the browser emits compatibility mouse events", () => { + LabelStudio.addFeatureFlagsOnPageLoad({ [FF_POLYGON_FREEHAND]: true }); + LabelStudio.params().config(imageToolsConfig).data(imageData).withResult([]).init(); + LabelStudio.waitForImageReady(); + ImageView.selectPolygonToolByButton(); + + drawFreehand("mouse", true); + + Sidebar.hasRegions(1); + Hotkeys.undo(); + Sidebar.hasNoRegions(); + }); }); From 589b10fc43d197341ba72bf3c1babc732f1bc72d Mon Sep 17 00:00:00 2001 From: ozymandiashh <234437643+ozymandiashh@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:55:31 +0000 Subject: [PATCH 3/3] fix: prune expired freehand compatibility guard and extend TTL The compatibility guard that suppresses the synthetic mouse event following a touch/pen freehand interaction was only time-checked, never cleared. If its synthetic event never arrived (e.g. the browser cancels the click after a sub-threshold cursor move between mousedown and mouseup), the stale guard could suppress one later, unrelated mouse event with nearby coordinates. Prune the guard as soon as it is found expired, extend the TTL from 500ms to 1500ms to comfortably cover the synthetic-event window, and route the timestamp through an injectable clock so the behaviour is unit-testable. --- .../src/components/ImageView/ImageView.jsx | 11 ++++++++--- .../ImageView/__tests__/ImageView.test.jsx | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/web/libs/editor/src/components/ImageView/ImageView.jsx b/web/libs/editor/src/components/ImageView/ImageView.jsx index 4f93cecad41e..730dfb250ee6 100644 --- a/web/libs/editor/src/components/ImageView/ImageView.jsx +++ b/web/libs/editor/src/components/ImageView/ImageView.jsx @@ -34,7 +34,7 @@ Konva.showWarnings = false; const hotkeys = Hotkey("Image"); const imgDefaultProps = { crossOrigin: "anonymous" }; -const FREEHAND_COMPATIBILITY_GUARD_MS = 500; +const FREEHAND_COMPATIBILITY_GUARD_MS = 1500; const FREEHAND_COMPATIBILITY_GUARD_DISTANCE = 25; const FREEHAND_DRAG_THRESHOLD = 5; @@ -543,6 +543,7 @@ export default observer( freehandTrace = []; freehandDragging = false; freehandCompatibilityGuard = null; + freehandCompatibilityClock = Date.now; constructor(props) { super(props); @@ -677,7 +678,7 @@ export default observer( this.freehandCompatibilityGuard = { clientX, clientY, - expiresAt: Date.now() + FREEHAND_COMPATIBILITY_GUARD_MS, + expiresAt: this.freehandCompatibilityClock() + FREEHAND_COMPATIBILITY_GUARD_MS, }; }; @@ -687,7 +688,11 @@ export default observer( const guard = this.freehandCompatibilityGuard; - if (!guard || Date.now() > guard.expiresAt) return false; + if (!guard) return false; + if (this.freehandCompatibilityClock() >= guard.expiresAt) { + this.freehandCompatibilityGuard = null; + return false; + } const clientX = Number.isFinite(event?.clientX) ? event.clientX : event?.offsetX; const clientY = Number.isFinite(event?.clientY) ? event.clientY : event?.offsetY; diff --git a/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx b/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx index f2794bbc29fd..4a91ce824a14 100644 --- a/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx +++ b/web/libs/editor/src/components/ImageView/__tests__/ImageView.test.jsx @@ -1362,6 +1362,23 @@ describe("ImageView with feature flags", () => { expect(item.event).toHaveBeenCalledWith("click", expect.anything(), 13, 10); }); + it("prunes a compatibility guard after its TTL", () => { + const { isFF } = require("../../../utils/feature-flags"); + const { view } = createFreehandPointerHarness(); + let now = 1000; + + isFF.mockImplementation((flag) => flag === "fflag_feat_front_polygon_freehand"); + view().freehandCompatibilityClock = () => now; + view().rememberFreehandCompatibilityEvent({ clientX: 10, clientY: 20 }); + + expect(view().freehandCompatibilityGuard.expiresAt).toBe(2500); + now = 2499; + expect(view().shouldSuppressFreehandCompatibilityEvent({ clientX: 10, clientY: 20 })).toBe(true); + now = 2501; + expect(view().shouldSuppressFreehandCompatibilityEvent({ clientX: 10, clientY: 20 })).toBe(false); + expect(view().freehandCompatibilityGuard).toBeNull(); + }); + it("does not start freehand drawing from an existing region or transformer", () => { const { isFF } = require("../../../utils/feature-flags"); const { commitFreehand, view } = createFreehandPointerHarness();