Fix absolute mouse position on multi-monitor targets + touch input

config.go

@@ -123,6 +123,9 @@ type Config struct {
 	NativeMaxRestart     uint                 `json:"native_max_restart_attempts"`
 	MqttConfig           *MQTTConfig          `json:"mqtt_config"`
 	AudioEnabled         bool                 `json:"audio_enabled"`
+	// AbsMouseMapping remaps absolute mouse coordinates onto the captured
+	// screen's rectangle within a multi-monitor desktop (nil = disabled).
+	AbsMouseMapping *AbsMouseMappingConfig `json:"abs_mouse_mapping,omitempty"`
 }
 
 // GetUpdateAPIURL returns the update API URL

jsonrpc.go

@@ -1350,6 +1350,8 @@ var rpcHandlers = map[string]RPCHandler{
 	"getJigglerState":            {Func: rpcGetJigglerState},
 	"setJigglerConfig":           {Func: rpcSetJigglerConfig, Params: []string{"jigglerConfig"}},
 	"getJigglerConfig":           {Func: rpcGetJigglerConfig},
+	"setAbsMouseMapping":         {Func: rpcSetAbsMouseMapping, Params: []string{"mapping"}},
+	"getAbsMouseMapping":         {Func: rpcGetAbsMouseMapping},
 	"getTimezones":               {Func: rpcGetTimezones},
 	"sendWOLMagicPacket":         {Func: rpcSendWOLMagicPacket, Params: []string{"macAddress"}, OptionalParams: []string{"broadcastIP"}},
 	"getStreamQualityFactor":     {Func: rpcGetStreamQualityFactor},

mouse_mapping.go

@@ -0,0 +1,172 @@
+package kvm
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"sync"
+
+	"github.com/gin-gonic/gin"
+)
+
+// AbsMouseMappingConfig maps absolute mouse coordinates onto a sub-rectangle
+// of the target's virtual desktop. The USB HID absolute pointer (0..32767) is
+// interpreted by the target OS across the bounding box of the *entire*
+// multi-monitor desktop, while JetKVM only captures a single screen. When
+// enabled, coordinates are remapped so that the video area corresponds to the
+// captured screen's rectangle within the full desktop.
+// All dimensions share one unit (use the target OS's logical pixels).
+type AbsMouseMappingConfig struct {
+	Enabled      bool `json:"enabled"`
+	TotalWidth   int  `json:"total_width"`
+	TotalHeight  int  `json:"total_height"`
+	ScreenX      int  `json:"screen_x"`
+	ScreenY      int  `json:"screen_y"`
+	ScreenWidth  int  `json:"screen_width"`
+	ScreenHeight int  `json:"screen_height"`
+}
+
+// maxMappingDimension bounds all mapping values: far beyond any real desktop,
+// small enough that int64 sums cannot overflow and float64 keeps precision.
+const maxMappingDimension = 1 << 20
+
+func (m *AbsMouseMappingConfig) validate() error {
+	if !m.Enabled {
+		return nil
+	}
+	if m.TotalWidth <= 0 || m.TotalHeight <= 0 {
+		return fmt.Errorf("total desktop size must be positive")
+	}
+	if m.ScreenWidth <= 0 || m.ScreenHeight <= 0 {
+		return fmt.Errorf("screen size must be positive")
+	}
+	if m.ScreenX < 0 || m.ScreenY < 0 {
+		return fmt.Errorf("screen position must not be negative")
+	}
+	if m.TotalWidth > maxMappingDimension || m.TotalHeight > maxMappingDimension ||
+		m.ScreenX > maxMappingDimension || m.ScreenY > maxMappingDimension ||
+		m.ScreenWidth > maxMappingDimension || m.ScreenHeight > maxMappingDimension {
+		return fmt.Errorf("mapping values must not exceed %d", maxMappingDimension)
+	}
+	// int64 arithmetic so absurd values can't wrap 32-bit int on arm builds
+	if int64(m.ScreenX)+int64(m.ScreenWidth) > int64(m.TotalWidth) ||
+		int64(m.ScreenY)+int64(m.ScreenHeight) > int64(m.TotalHeight) {
+		return fmt.Errorf("screen rectangle must fit within the total desktop")
+	}
+	return nil
+}
+
+func clampAbsCoord(v float64) int {
+	i := int(math.Round(v))
+	if i < 0 {
+		return 0
+	}
+	if i > 32767 {
+		return 32767
+	}
+	return i
+}
+
+// absMouseMappingLock guards the config.AbsMouseMapping pointer. The setter
+// always installs a freshly-validated struct and never mutates in place, so
+// readers only need a consistent pointer snapshot.
+var absMouseMappingLock sync.RWMutex
+
+// absMouseMappingUpdateMu serializes whole set operations (swap + save +
+// rollback) so a failing update cannot roll back over a newer successful one.
+var absMouseMappingUpdateMu sync.Mutex
+
+func currentAbsMouseMapping() *AbsMouseMappingConfig {
+	absMouseMappingLock.RLock()
+	defer absMouseMappingLock.RUnlock()
+	if config == nil {
+		return nil
+	}
+	return config.AbsMouseMapping
+}
+
+// applyAbsMouseMapping remaps x/y (0..32767 relative to the captured screen)
+// into 0..32767 relative to the target's full desktop bounding box.
+func applyAbsMouseMapping(x int, y int) (int, int) {
+	m := currentAbsMouseMapping()
+	if m == nil || !m.Enabled {
+		return x, y
+	}
+	if m.validate() != nil {
+		return x, y
+	}
+	nx := (float64(m.ScreenX) + float64(x)/32767.0*float64(m.ScreenWidth)) / float64(m.TotalWidth) * 32767.0
+	ny := (float64(m.ScreenY) + float64(y)/32767.0*float64(m.ScreenHeight)) / float64(m.TotalHeight) * 32767.0
+	return clampAbsCoord(nx), clampAbsCoord(ny)
+}
+
+func rpcGetAbsMouseMapping() (AbsMouseMappingConfig, error) {
+	m := currentAbsMouseMapping()
+	if m == nil {
+		return AbsMouseMappingConfig{}, nil
+	}
+	return *m, nil
+}
+
+func rpcSetAbsMouseMapping(mapping AbsMouseMappingConfig) error {
+	if err := mapping.validate(); err != nil {
+		return err
+	}
+	absMouseMappingUpdateMu.Lock()
+	defer absMouseMappingUpdateMu.Unlock()
+	absMouseMappingLock.Lock()
+	if config == nil {
+		absMouseMappingLock.Unlock()
+		return fmt.Errorf("config not loaded")
+	}
+	previous := config.AbsMouseMapping
+	config.AbsMouseMapping = &mapping
+	absMouseMappingLock.Unlock()
+	if err := SaveConfig(); err != nil {
+		// Keep in-memory state consistent with what the caller was told:
+		// the update failed, so restore the previous mapping.
+		absMouseMappingLock.Lock()
+		config.AbsMouseMapping = previous
+		absMouseMappingLock.Unlock()
+		return fmt.Errorf("failed to save config: %w", err)
+	}
+	logger.Info().
+		Bool("enabled", mapping.Enabled).
+		Int("totalWidth", mapping.TotalWidth).
+		Int("totalHeight", mapping.TotalHeight).
+		Int("screenX", mapping.ScreenX).
+		Int("screenY", mapping.ScreenY).
+		Int("screenWidth", mapping.ScreenWidth).
+		Int("screenHeight", mapping.ScreenHeight).
+		Msg("absolute mouse mapping updated")
+	return nil
+}
+
+// HTTP endpoints (authenticated) so host-side automation can keep the mapping
+// in sync with display layout changes without a WebRTC session.
+func handleGetMouseMapping(c *gin.Context) {
+	m := currentAbsMouseMapping()
+	if m == nil {
+		c.JSON(http.StatusOK, AbsMouseMappingConfig{})
+		return
+	}
+	c.JSON(http.StatusOK, *m)
+}
+
+func handleSetMouseMapping(c *gin.Context) {
+	var mapping AbsMouseMappingConfig
+	if err := c.ShouldBindJSON(&mapping); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	// Client errors (invalid mapping) are 400; persistence failures are 500.
+	if err := mapping.validate(); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	if err := rpcSetAbsMouseMapping(mapping); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"status": "ok"})
+}

mouse_mapping_test.go

@@ -0,0 +1,140 @@
+package kvm
+
+import "testing"
+
+func TestAbsMouseMappingValidate(t *testing.T) {
+	tests := []struct {
+		name    string
+		mapping AbsMouseMappingConfig
+		wantErr bool
+	}{
+		{"disabled is always valid", AbsMouseMappingConfig{Enabled: false}, false},
+		{"valid interior screen", AbsMouseMappingConfig{Enabled: true, TotalWidth: 5263, TotalHeight: 1080, ScreenX: 3343, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}, false},
+		{"screen equals desktop", AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenX: 0, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}, false},
+		{"zero total", AbsMouseMappingConfig{Enabled: true, TotalWidth: 0, TotalHeight: 1080, ScreenWidth: 1920, ScreenHeight: 1080}, true},
+		{"zero screen", AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenWidth: 0, ScreenHeight: 1080}, true},
+		{"negative position", AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenX: -1, ScreenWidth: 1920, ScreenHeight: 1080}, true},
+		{"screen exceeds desktop", AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenX: 1, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}, true},
+		// Rejected by the sanity bound (which also renders the int64 fit
+		// check unreachable for wrapping-scale values — defense in depth).
+		{"absurd int32-scale values rejected", AbsMouseMappingConfig{Enabled: true, TotalWidth: 2000000000, TotalHeight: 1080, ScreenX: 2000000000, ScreenY: 0, ScreenWidth: 2000000000, ScreenHeight: 1080}, true},
+		{"dimension above sanity bound rejected", AbsMouseMappingConfig{Enabled: true, TotalWidth: maxMappingDimension + 1, TotalHeight: 1080, ScreenX: 0, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}, true},
+		{"dimension at sanity bound accepted", AbsMouseMappingConfig{Enabled: true, TotalWidth: maxMappingDimension, TotalHeight: maxMappingDimension, ScreenX: 0, ScreenY: 0, ScreenWidth: maxMappingDimension, ScreenHeight: maxMappingDimension}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.mapping.validate()
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("validate() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestApplyAbsMouseMapping(t *testing.T) {
+	// The user-facing contract: video position fraction f maps to desktop
+	// position screenX + f*screenW, expressed as a fraction of the desktop.
+	withMapping := func(m *AbsMouseMappingConfig, fn func()) {
+		absMouseMappingLock.Lock()
+		if config == nil {
+			config = &Config{}
+		}
+		previous := config.AbsMouseMapping
+		config.AbsMouseMapping = m
+		absMouseMappingLock.Unlock()
+		defer func() {
+			absMouseMappingLock.Lock()
+			config.AbsMouseMapping = previous
+			absMouseMappingLock.Unlock()
+		}()
+		fn()
+	}
+
+	t.Run("nil mapping is identity", func(t *testing.T) {
+		withMapping(nil, func() {
+			for _, v := range []int{0, 1, 16384, 32766, 32767} {
+				if x, y := applyAbsMouseMapping(v, v); x != v || y != v {
+					t.Fatalf("identity violated: %d -> (%d,%d)", v, x, y)
+				}
+			}
+		})
+	})
+
+	t.Run("disabled mapping is identity", func(t *testing.T) {
+		withMapping(&AbsMouseMappingConfig{Enabled: false, TotalWidth: 5263, TotalHeight: 1080, ScreenX: 3343, ScreenWidth: 1920, ScreenHeight: 1080}, func() {
+			if x, y := applyAbsMouseMapping(100, 200); x != 100 || y != 200 {
+				t.Fatalf("disabled mapping altered coords: (%d,%d)", x, y)
+			}
+		})
+	})
+
+	t.Run("invalid enabled mapping falls back to identity", func(t *testing.T) {
+		withMapping(&AbsMouseMappingConfig{Enabled: true, TotalWidth: 0, TotalHeight: 0, ScreenWidth: 0, ScreenHeight: 0}, func() {
+			if x, y := applyAbsMouseMapping(100, 200); x != 100 || y != 200 {
+				t.Fatalf("invalid mapping altered coords: (%d,%d)", x, y)
+			}
+		})
+	})
+
+	t.Run("three-screen layout maps into captured screen", func(t *testing.T) {
+		m := &AbsMouseMappingConfig{Enabled: true, TotalWidth: 5263, TotalHeight: 1080, ScreenX: 3343, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}
+		withMapping(m, func() {
+			cases := []struct{ in, wantX, wantY int }{
+				// left edge of video -> left edge of captured screen
+				{0, 20813, 0}, // round(3343/5263*32767)
+				// center -> screen center within desktop
+				{16384, 26790, 16384}, // round((3343+960.03)/5263*32767)
+				// right edge -> right edge of desktop (screen is rightmost)
+				{32767, 32767, 32767},
+			}
+			for _, c := range cases {
+				x, y := applyAbsMouseMapping(c.in, c.in)
+				if x != c.wantX {
+					t.Errorf("x: %d -> %d, want %d", c.in, x, c.wantX)
+				}
+				if y != c.wantY {
+					t.Errorf("y: %d -> %d, want %d", c.in, y, c.wantY)
+				}
+			}
+		})
+	})
+
+	t.Run("screen equals desktop is near-identity", func(t *testing.T) {
+		m := &AbsMouseMappingConfig{Enabled: true, TotalWidth: 1920, TotalHeight: 1080, ScreenX: 0, ScreenY: 0, ScreenWidth: 1920, ScreenHeight: 1080}
+		withMapping(m, func() {
+			for _, v := range []int{0, 1, 12345, 32766, 32767} {
+				if x, _ := applyAbsMouseMapping(v, v); x != v {
+					t.Fatalf("identity-shaped mapping moved %d to %d", v, x)
+				}
+			}
+		})
+	})
+
+	t.Run("output clamped to valid range", func(t *testing.T) {
+		m := &AbsMouseMappingConfig{Enabled: true, TotalWidth: 100, TotalHeight: 100, ScreenX: 0, ScreenY: 0, ScreenWidth: 100, ScreenHeight: 100}
+		withMapping(m, func() {
+			// Inputs outside 0..32767 (defensive: hidrpc decodes uint16)
+			if x, _ := applyAbsMouseMapping(40000, 0); x != 32767 {
+				t.Fatalf("clamp high failed: %d", x)
+			}
+			if x, _ := applyAbsMouseMapping(-5, 0); x != 0 {
+				t.Fatalf("clamp low failed: %d", x)
+			}
+		})
+	})
+}
+
+func TestClampAbsCoord(t *testing.T) {
+	cases := []struct {
+		in   float64
+		want int
+	}{
+		{-1, 0}, {0, 0}, {0.4, 0}, {0.5, 1}, {16383.6, 16384},
+		{32766.5, 32767}, {32767, 32767}, {40000, 32767},
+	}
+	for _, c := range cases {
+		if got := clampAbsCoord(c.in); got != c.want {
+			t.Errorf("clampAbsCoord(%v) = %d, want %d", c.in, got, c.want)
+		}
+	}
+}

ui/localization/messages/en.json

@@ -648,6 +648,18 @@
     "mouse_mode_relative_description": "Most compatible",
     "mouse_modes_description": "Choose the mouse input mode",
     "mouse_modes_title": "Modes",
+    "mouse_multimonitor_description": "Fix absolute mouse offset when the target machine has multiple displays",
+    "mouse_multimonitor_error": "Failed to update multi-monitor mapping: {error}",
+    "mouse_multimonitor_hint": "Values are in the target OS's logical pixels. Total = bounding box of all displays combined; screen = position and size of the display captured by JetKVM within it.",
+    "mouse_multimonitor_save": "Save Mapping",
+    "mouse_multimonitor_screen_height_label": "Captured Screen Height",
+    "mouse_multimonitor_screen_width_label": "Captured Screen Width",
+    "mouse_multimonitor_screen_x_label": "Captured Screen X",
+    "mouse_multimonitor_screen_y_label": "Captured Screen Y",
+    "mouse_multimonitor_title": "Multi-Monitor Mapping",
+    "mouse_multimonitor_total_height_label": "Total Desktop Height",
+    "mouse_multimonitor_total_width_label": "Total Desktop Width",
+    "mouse_multimonitor_updated": "Multi-monitor mapping updated",
     "mouse_scroll_high": "High",
     "mouse_scroll_invert_description": "Enable if your host machine scrolls in the wrong direction",
     "mouse_scroll_invert_title": "Invert Scroll Direction",