diff --git a/config.go b/config.go index 32b3b659b..e7ff02426 100644 --- a/config.go +++ b/config.go @@ -123,6 +123,9 @@ type Config struct { NativeMaxRestart uint `json:"native_max_restart_attempts"` MqttConfig *MQTTConfig `json:"mqtt_config"` AudioEnabled bool `json:"audio_enabled"` + // AbsMouseMapping remaps absolute mouse coordinates onto the captured + // screen's rectangle within a multi-monitor desktop (nil = disabled). + AbsMouseMapping *AbsMouseMappingConfig `json:"abs_mouse_mapping,omitempty"` } // GetUpdateAPIURL returns the update API URL diff --git a/jsonrpc.go b/jsonrpc.go index 5a662530d..7d44e103e 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1350,6 +1350,8 @@ var rpcHandlers = map[string]RPCHandler{ "getJigglerState": {Func: rpcGetJigglerState}, "setJigglerConfig": {Func: rpcSetJigglerConfig, Params: []string{"jigglerConfig"}}, "getJigglerConfig": {Func: rpcGetJigglerConfig}, + "setAbsMouseMapping": {Func: rpcSetAbsMouseMapping, Params: []string{"mapping"}}, + "getAbsMouseMapping": {Func: rpcGetAbsMouseMapping}, "getTimezones": {Func: rpcGetTimezones}, "sendWOLMagicPacket": {Func: rpcSendWOLMagicPacket, Params: []string{"macAddress"}, OptionalParams: []string{"broadcastIP"}}, "getStreamQualityFactor": {Func: rpcGetStreamQualityFactor}, diff --git a/mouse_mapping.go b/mouse_mapping.go new file mode 100644 index 000000000..4939b0c94 --- /dev/null +++ b/mouse_mapping.go @@ -0,0 +1,172 @@ +package kvm + +import ( + "fmt" + "math" + "net/http" + "sync" + + "github.com/gin-gonic/gin" +) + +// AbsMouseMappingConfig maps absolute mouse coordinates onto a sub-rectangle +// of the target's virtual desktop. The USB HID absolute pointer (0..32767) is +// interpreted by the target OS across the bounding box of the *entire* +// multi-monitor desktop, while JetKVM only captures a single screen. When +// enabled, coordinates are remapped so that the video area corresponds to the +// captured screen's rectangle within the full desktop. +// All dimensions share one unit (use the target OS's logical pixels). +type AbsMouseMappingConfig struct { + Enabled bool `json:"enabled"` + TotalWidth int `json:"total_width"` + TotalHeight int `json:"total_height"` + ScreenX int `json:"screen_x"` + ScreenY int `json:"screen_y"` + ScreenWidth int `json:"screen_width"` + ScreenHeight int `json:"screen_height"` +} + +// maxMappingDimension bounds all mapping values: far beyond any real desktop, +// small enough that int64 sums cannot overflow and float64 keeps precision. +const maxMappingDimension = 1 << 20 + +func (m *AbsMouseMappingConfig) validate() error { + if !m.Enabled { + return nil + } + if m.TotalWidth <= 0 || m.TotalHeight <= 0 { + return fmt.Errorf("total desktop size must be positive") + } + if m.ScreenWidth <= 0 || m.ScreenHeight <= 0 { + return fmt.Errorf("screen size must be positive") + } + if m.ScreenX < 0 || m.ScreenY < 0 { + return fmt.Errorf("screen position must not be negative") + } + if m.TotalWidth > maxMappingDimension || m.TotalHeight > maxMappingDimension || + m.ScreenX > maxMappingDimension || m.ScreenY > maxMappingDimension || + m.ScreenWidth > maxMappingDimension || m.ScreenHeight > maxMappingDimension { + return fmt.Errorf("mapping values must not exceed %d", maxMappingDimension) + } + // int64 arithmetic so absurd values can't wrap 32-bit int on arm builds + if int64(m.ScreenX)+int64(m.ScreenWidth) > int64(m.TotalWidth) || + int64(m.ScreenY)+int64(m.ScreenHeight) > int64(m.TotalHeight) { + return fmt.Errorf("screen rectangle must fit within the total desktop") + } + return nil +} + +func clampAbsCoord(v float64) int { + i := int(math.Round(v)) + if i < 0 { + return 0 + } + if i > 32767 { + return 32767 + } + return i +} + +// absMouseMappingLock guards the config.AbsMouseMapping pointer. The setter +// always installs a freshly-validated struct and never mutates in place, so +// readers only need a consistent pointer snapshot. +var absMouseMappingLock sync.RWMutex + +// absMouseMappingUpdateMu serializes whole set operations (swap + save + +// rollback) so a failing update cannot roll back over a newer successful one. +var absMouseMappingUpdateMu sync.Mutex + +func currentAbsMouseMapping() *AbsMouseMappingConfig { + absMouseMappingLock.RLock() + defer absMouseMappingLock.RUnlock() + if config == nil { + return nil + } + return config.AbsMouseMapping +} + +// applyAbsMouseMapping remaps x/y (0..32767 relative to the captured screen) +// into 0..32767 relative to the target's full desktop bounding box. +func applyAbsMouseMapping(x int, y int) (int, int) { + m := currentAbsMouseMapping() + if m == nil || !m.Enabled { + return x, y + } + if m.validate() != nil { + return x, y + } + nx := (float64(m.ScreenX) + float64(x)/32767.0*float64(m.ScreenWidth)) / float64(m.TotalWidth) * 32767.0 + ny := (float64(m.ScreenY) + float64(y)/32767.0*float64(m.ScreenHeight)) / float64(m.TotalHeight) * 32767.0 + return clampAbsCoord(nx), clampAbsCoord(ny) +} + +func rpcGetAbsMouseMapping() (AbsMouseMappingConfig, error) { + m := currentAbsMouseMapping() + if m == nil { + return AbsMouseMappingConfig{}, nil + } + return *m, nil +} + +func rpcSetAbsMouseMapping(mapping AbsMouseMappingConfig) error { + if err := mapping.validate(); err != nil { + return err + } + absMouseMappingUpdateMu.Lock() + defer absMouseMappingUpdateMu.Unlock() + absMouseMappingLock.Lock() + if config == nil { + absMouseMappingLock.Unlock() + return fmt.Errorf("config not loaded") + } + previous := config.AbsMouseMapping + config.AbsMouseMapping = &mapping + absMouseMappingLock.Unlock() + if err := SaveConfig(); err != nil { + // Keep in-memory state consistent with what the caller was told: + // the update failed, so restore the previous mapping. + absMouseMappingLock.Lock() + config.AbsMouseMapping = previous + absMouseMappingLock.Unlock() + return fmt.Errorf("failed to save config: %w", err) + } + logger.Info(). + Bool("enabled", mapping.Enabled). + Int("totalWidth", mapping.TotalWidth). + Int("totalHeight", mapping.TotalHeight). + Int("screenX", mapping.ScreenX). + Int("screenY", mapping.ScreenY). + Int("screenWidth", mapping.ScreenWidth). + Int("screenHeight", mapping.ScreenHeight). + Msg("absolute mouse mapping updated") + return nil +} + +// HTTP endpoints (authenticated) so host-side automation can keep the mapping +// in sync with display layout changes without a WebRTC session. +func handleGetMouseMapping(c *gin.Context) { + m := currentAbsMouseMapping() + if m == nil { + c.JSON(http.StatusOK, AbsMouseMappingConfig{}) + return + } + c.JSON(http.StatusOK, *m) +} + +func handleSetMouseMapping(c *gin.Context) { + var mapping AbsMouseMappingConfig + if err := c.ShouldBindJSON(&mapping); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + // Client errors (invalid mapping) are 400; persistence failures are 500. + if err := mapping.validate(); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + if err := rpcSetAbsMouseMapping(mapping); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, gin.H{"status": "ok"}) +} diff --git a/mouse_mapping_test.go b/mouse_mapping_test.go new file mode 100644 index 000000000..b8b07f251 --- /dev/null +++ b/mouse_mapping_test.go @@ -0,0 +1,140 @@ +package kvm + +import "testing" + +func TestAbsMouseMappingValidate(t *testing.T) { + tests := []struct { + name string + mapping AbsMouseMappingConfig + wantErr bool + }{ + {"disabled is always valid", AbsMouseMappingConfig{Enabled: false}, false}, + {"valid interior screen", AbsMouseMappingConfig{Enabled: true, TotalWidth: 5263, TotalHeight: 1080, ScreenX: 3343, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}, false}, + {"screen equals desktop", AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenX: 0, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}, false}, + {"zero total", AbsMouseMappingConfig{Enabled: true, TotalWidth: 0, TotalHeight: 1080, ScreenWidth: 1920, ScreenHeight: 1080}, true}, + {"zero screen", AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenWidth: 0, ScreenHeight: 1080}, true}, + {"negative position", AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenX: -1, ScreenWidth: 1920, ScreenHeight: 1080}, true}, + {"screen exceeds desktop", AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenX: 1, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}, true}, + // Rejected by the sanity bound (which also renders the int64 fit + // check unreachable for wrapping-scale values — defense in depth). + {"absurd int32-scale values rejected", AbsMouseMappingConfig{Enabled: true, TotalWidth: 2000000000, TotalHeight: 1080, ScreenX: 2000000000, ScreenY: 0, ScreenWidth: 2000000000, ScreenHeight: 1080}, true}, + {"dimension above sanity bound rejected", AbsMouseMappingConfig{Enabled: true, TotalWidth: maxMappingDimension + 1, TotalHeight: 1080, ScreenX: 0, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}, true}, + {"dimension at sanity bound accepted", AbsMouseMappingConfig{Enabled: true, TotalWidth: maxMappingDimension, TotalHeight: maxMappingDimension, ScreenX: 0, ScreenY: 0, ScreenWidth: maxMappingDimension, ScreenHeight: maxMappingDimension}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.mapping.validate() + if (err != nil) != tt.wantErr { + t.Fatalf("validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestApplyAbsMouseMapping(t *testing.T) { + // The user-facing contract: video position fraction f maps to desktop + // position screenX + f*screenW, expressed as a fraction of the desktop. + withMapping := func(m *AbsMouseMappingConfig, fn func()) { + absMouseMappingLock.Lock() + if config == nil { + config = &Config{} + } + previous := config.AbsMouseMapping + config.AbsMouseMapping = m + absMouseMappingLock.Unlock() + defer func() { + absMouseMappingLock.Lock() + config.AbsMouseMapping = previous + absMouseMappingLock.Unlock() + }() + fn() + } + + t.Run("nil mapping is identity", func(t *testing.T) { + withMapping(nil, func() { + for _, v := range []int{0, 1, 16384, 32766, 32767} { + if x, y := applyAbsMouseMapping(v, v); x != v || y != v { + t.Fatalf("identity violated: %d -> (%d,%d)", v, x, y) + } + } + }) + }) + + t.Run("disabled mapping is identity", func(t *testing.T) { + withMapping(&AbsMouseMappingConfig{Enabled: false, TotalWidth: 5263, TotalHeight: 1080, ScreenX: 3343, ScreenWidth: 1920, ScreenHeight: 1080}, func() { + if x, y := applyAbsMouseMapping(100, 200); x != 100 || y != 200 { + t.Fatalf("disabled mapping altered coords: (%d,%d)", x, y) + } + }) + }) + + t.Run("invalid enabled mapping falls back to identity", func(t *testing.T) { + withMapping(&AbsMouseMappingConfig{Enabled: true, TotalWidth: 0, TotalHeight: 0, ScreenWidth: 0, ScreenHeight: 0}, func() { + if x, y := applyAbsMouseMapping(100, 200); x != 100 || y != 200 { + t.Fatalf("invalid mapping altered coords: (%d,%d)", x, y) + } + }) + }) + + t.Run("three-screen layout maps into captured screen", func(t *testing.T) { + m := &AbsMouseMappingConfig{Enabled: true, TotalWidth: 5263, TotalHeight: 1080, ScreenX: 3343, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080} + withMapping(m, func() { + cases := []struct{ in, wantX, wantY int }{ + // left edge of video -> left edge of captured screen + {0, 20813, 0}, // round(3343/5263*32767) + // center -> screen center within desktop + {16384, 26790, 16384}, // round((3343+960.03)/5263*32767) + // right edge -> right edge of desktop (screen is rightmost) + {32767, 32767, 32767}, + } + for _, c := range cases { + x, y := applyAbsMouseMapping(c.in, c.in) + if x != c.wantX { + t.Errorf("x: %d -> %d, want %d", c.in, x, c.wantX) + } + if y != c.wantY { + t.Errorf("y: %d -> %d, want %d", c.in, y, c.wantY) + } + } + }) + }) + + t.Run("screen equals desktop is near-identity", func(t *testing.T) { + m := &AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenX: 0, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080} + withMapping(m, func() { + for _, v := range []int{0, 1, 12345, 32766, 32767} { + if x, _ := applyAbsMouseMapping(v, v); x != v { + t.Fatalf("identity-shaped mapping moved %d to %d", v, x) + } + } + }) + }) + + t.Run("output clamped to valid range", func(t *testing.T) { + m := &AbsMouseMappingConfig{Enabled: true, TotalWidth: 100, TotalHeight: 100, ScreenX: 0, ScreenY: 0, ScreenWidth: 100, ScreenHeight: 100} + withMapping(m, func() { + // Inputs outside 0..32767 (defensive: hidrpc decodes uint16) + if x, _ := applyAbsMouseMapping(40000, 0); x != 32767 { + t.Fatalf("clamp high failed: %d", x) + } + if x, _ := applyAbsMouseMapping(-5, 0); x != 0 { + t.Fatalf("clamp low failed: %d", x) + } + }) + }) +} + +func TestClampAbsCoord(t *testing.T) { + cases := []struct { + in float64 + want int + }{ + {-1, 0}, {0, 0}, {0.4, 0}, {0.5, 1}, {16383.6, 16384}, + {32766.5, 32767}, {32767, 32767}, {40000, 32767}, + } + for _, c := range cases { + if got := clampAbsCoord(c.in); got != c.want { + t.Errorf("clampAbsCoord(%v) = %d, want %d", c.in, got, c.want) + } + } +} diff --git a/ui/localization/messages/en.json b/ui/localization/messages/en.json index 63be6e8d9..05a0665bb 100644 --- a/ui/localization/messages/en.json +++ b/ui/localization/messages/en.json @@ -648,6 +648,18 @@ "mouse_mode_relative_description": "Most compatible", "mouse_modes_description": "Choose the mouse input mode", "mouse_modes_title": "Modes", + "mouse_multimonitor_description": "Fix absolute mouse offset when the target machine has multiple displays", + "mouse_multimonitor_error": "Failed to update multi-monitor mapping: {error}", + "mouse_multimonitor_hint": "Values are in the target OS's logical pixels. Total = bounding box of all displays combined; screen = position and size of the display captured by JetKVM within it.", + "mouse_multimonitor_save": "Save Mapping", + "mouse_multimonitor_screen_height_label": "Captured Screen Height", + "mouse_multimonitor_screen_width_label": "Captured Screen Width", + "mouse_multimonitor_screen_x_label": "Captured Screen X", + "mouse_multimonitor_screen_y_label": "Captured Screen Y", + "mouse_multimonitor_title": "Multi-Monitor Mapping", + "mouse_multimonitor_total_height_label": "Total Desktop Height", + "mouse_multimonitor_total_width_label": "Total Desktop Width", + "mouse_multimonitor_updated": "Multi-monitor mapping updated", "mouse_scroll_high": "High", "mouse_scroll_invert_description": "Enable if your host machine scrolls in the wrong direction", "mouse_scroll_invert_title": "Invert Scroll Direction", diff --git a/ui/src/components/MouseMappingSetting.tsx b/ui/src/components/MouseMappingSetting.tsx new file mode 100644 index 000000000..4782b5eba --- /dev/null +++ b/ui/src/components/MouseMappingSetting.tsx @@ -0,0 +1,78 @@ +import { useEffect, useState } from "react"; + +import { Button } from "@components/Button"; +import { InputFieldWithLabel } from "@components/InputField"; +import { m } from "@localizations/messages.js"; + +export interface AbsMouseMapping { + enabled: boolean; + total_width: number; + total_height: number; + screen_x: number; + screen_y: number; + screen_width: number; + screen_height: number; +} + +export const defaultAbsMouseMapping: AbsMouseMapping = { + enabled: false, + total_width: 0, + total_height: 0, + screen_x: 0, + screen_y: 0, + screen_width: 0, + screen_height: 0, +}; + +export function MouseMappingSetting({ + mapping, + onSave, +}: { + mapping: AbsMouseMapping; + onSave: (mapping: AbsMouseMapping) => void; +}) { + const [state, setState] = useState(mapping); + + useEffect(() => { + setState(mapping); + }, [mapping]); + + const numberField = (key: keyof Omit, label: string, min: number) => ( + { + // Guard the controlled input against NaN (empty/partial input -> 0, + // which the device-side validation rejects with a clear error). + const n = Number(e.target.value); + setState({ ...state, [key]: Number.isFinite(n) ? Math.trunc(n) : 0 }); + }} + /> + ); + + return ( +
+

{m.mouse_multimonitor_hint()}

+
+ {numberField("total_width", m.mouse_multimonitor_total_width_label(), 1)} + {numberField("total_height", m.mouse_multimonitor_total_height_label(), 1)} + {numberField("screen_x", m.mouse_multimonitor_screen_x_label(), 0)} + {numberField("screen_y", m.mouse_multimonitor_screen_y_label(), 0)} + {numberField("screen_width", m.mouse_multimonitor_screen_width_label(), 1)} + {numberField("screen_height", m.mouse_multimonitor_screen_height_label(), 1)} +
+
+
+
+ ); +} diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 36a4be61f..def525b95 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -61,8 +61,6 @@ export default function WebRTCVideo({ setSize: setVideoSize, width: videoWidth, height: videoHeight, - clientWidth: videoClientWidth, - clientHeight: videoClientHeight, hdmiState, setVideoElement, setContainerElement, @@ -282,17 +280,18 @@ export default function WebRTCVideo({ }; document.addEventListener("fullscreenchange", handleFullscreenChange); + return () => { + document.removeEventListener("fullscreenchange", handleFullscreenChange); + }; }, [releaseKeyboardLock]); const absMouseMoveHandler = useMemo( () => getAbsMouseMoveHandler({ - videoClientWidth, - videoClientHeight, videoWidth, videoHeight, }), - [getAbsMouseMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight], + [getAbsMouseMoveHandler, videoWidth, videoHeight], ); const relMouseMoveHandler = useMemo(() => getRelMouseMoveHandler(), [getRelMouseMoveHandler]); @@ -581,7 +580,11 @@ export default function WebRTCVideo({ const abortController = new AbortController(); const signal = abortController.signal; - videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal }); + // Absolute mode listens to pointer events so touch input (iPad/tablet) + // is handled natively, incl. move events while dragging. Relative mode + // keeps mousemove, which is what pointer lock delivers movement through. + const moveEventName = isRelativeMouseMode ? "mousemove" : "pointermove"; + videoElmRefValue.addEventListener(moveEventName, mouseHandler, { signal }); videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal }); videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal }); videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { @@ -603,6 +606,23 @@ export default function WebRTCVideo({ // Reset the mouse position when the window is blurred or the document is hidden window.addEventListener("blur", resetMousePosition, { signal }); document.addEventListener("visibilitychange", resetMousePosition, { signal }); + // Release buttons if a touch interaction is canceled by the system + // (e.g. iPadOS edge gestures), so buttons don't get stuck down. + videoElmRefValue.addEventListener("pointercancel", resetMousePosition, { signal }); + // Capture the pointer while a button is held so the release still + // reaches us when it happens outside the video element (touch + // pointers are implicitly captured; this covers mouse/pen). + videoElmRefValue.addEventListener( + "pointerdown", + (e: PointerEvent) => { + try { + videoElmRefValue.setPointerCapture(e.pointerId); + } catch { + // ignore: capture is best-effort (e.g. pointer already gone) + } + }, + { signal }, + ); } const preventContextMenu = (e: MouseEvent) => e.preventDefault(); @@ -721,17 +741,23 @@ export default function WebRTCVideo({ disablePictureInPicture controlsList="nofullscreen" style={videoStyle} - className={cx("h-full w-full object-contain transition-all duration-1000", { - "cursor-none": settings.isCursorHidden, - "pointer-events-none": isOcrMode, - "opacity-0!": - isVideoLoading || - hdmiError || - hasConnectionIssues || - peerConnectionState !== "connected", - "opacity-60!": showPointerLockBar, - "animate-slideUpFade": isPlaying, - })} + className={cx( + // touch-none: the video is the interaction surface — + // deliver touch input as pointer events instead of + // letting the browser scroll/zoom the page with it. + "h-full w-full touch-none object-contain transition-all duration-1000", + { + "cursor-none": settings.isCursorHidden, + "pointer-events-none": isOcrMode, + "opacity-0!": + isVideoLoading || + hdmiError || + hasConnectionIssues || + peerConnectionState !== "connected", + "opacity-60!": showPointerLockBar, + "animate-slideUpFade": isPlaying, + }, + )} /> {audioEnabled &&