Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
184 changes: 184 additions & 0 deletions electron/ipc/cursorTelemetryLifecycle.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
import fs from "node:fs";
import path from "node:path";
import { runInNewContext } from "node:vm";
import ts from "typescript";
import { describe, expect, it, vi } from "vitest";
import {
compactCursorTelemetryPauseRanges,
finalizeSampledCursorTelemetry,
} from "../native-bridge/cursor/recording/cursorPauseRanges";

const source = fs.readFileSync(new URL("./handlers.ts", import.meta.url), "utf8");

function execute(code: string, context: object) {
runInNewContext(
ts.transpileModule(code, {
compilerOptions: { target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.None },
}).outputText,
context,
);
}

describe("native Linux fallback timing", () => {
it("starts the sampler on the helper pause-range clock, not picker or delayed sampler time", async () => {
const start = source.indexOf("async function startCursorRecording(");
const end = source.indexOf("\nasync function stopCursorRecording", start);
const create = vi.fn((_options: { startTimeMs?: number }) => ({
start: vi.fn(async () => undefined),
}));
const context = {
linuxNativeCaptureSession: { cursorStartTimeMs: 1000 },
isX11PointerFallbackEnabled: () => true,
cursorRecordingSession: null,
pendingCursorRecordingData: null,
createX11PointerRecordingSession: create,
getX11CaptureBounds: () => null,
MAX_CURSOR_SAMPLES: 100,
CURSOR_SAMPLE_INTERVAL_MS: 33,
console: { info: vi.fn() },
startCursorRecording: undefined as unknown as (id: number) => Promise<void>,
};
execute(source.slice(start, end), context);
await context.startCursorRecording(100);
expect(create.mock.calls[0][0]).toMatchObject({ startTimeMs: 1000 });
});
function fixture(mode = "editable-overlay", helperHasSamples = false) {
const start = source.indexOf('ipcMain.handle("stop-native-linux-recording"');
const end = source.indexOf("\n\t});", start) + 7;
const handle = vi.fn();
let finish!: () => void;
const done = new Promise<void>((resolve) => {
finish = resolve;
});
const cursor = {
version: 2 as const,
provider: "none" as const,
assets: [],
samples: (helperHasSamples ? [42] : []).map((timeMs) => ({ timeMs, cx: 0.5, cy: 0.5 })),
};
const result = {
path: "/take.mp4",
durationMs: 500,
interrupted: false,
cursor,
cursorPauseRanges: [{ startMs: 200, endMs: 400 }],
};
const session = {
stop: vi.fn(async () => {
await done;
return result;
}),
discard: vi.fn(),
};
const context = {
ipcMain: { handle },
linuxNativeCaptureSession: session,
linuxNativeCaptureRecordingId: 7,
linuxNativeCaptureCursorMode: mode,
pendingCursorRecordingData: null as unknown,
stopCursorRecording: vi.fn(async () => {
context.pendingCursorRecordingData = {
...cursor,
samples: [100, 200, 300, 400, 500, 700, 701, 1500].map((timeMs) => ({
timeMs,
cx: 0.5,
cy: 0.5,
})),
};
}),
fs: {
writeFile: vi.fn(async (_path: string, _body: string) => undefined),
rm: vi.fn(async () => undefined),
},
compactCursorTelemetryPauseRanges,
finalizeSampledCursorTelemetry,
path,
RECORDINGS_DIR: "/recordings",
RECORDING_FILE_PREFIX: "recording-",
RECORDING_SESSION_SUFFIX: ".json",
setCurrentRecordingSessionState: vi.fn(),
currentProjectPath: null,
nativeRecordingWarning: null,
mainT: () => "interrupted",
formatDuration: () => "0:00",
registerRecordingMediaLinks: vi.fn(),
console: { info: vi.fn(), error: vi.fn() },
linuxNativeCaptureSourceLabel: null,
linuxSourceLabel: () => "screen",
setAppRecording: vi.fn(),
};
execute(source.slice(start, end), context);
return {
context,
session,
finish,
result,
stop: handle.mock.calls[0][1] as (e: unknown, discard?: boolean) => Promise<unknown>,
};
}
it("stops the sampler before helper teardown completes and bounds compacted samples to video", async () => {
const { context, finish, stop } = fixture();
const pending = stop({});
const stoppedBeforeHelper = context.stopCursorRecording.mock.calls.length;
finish();
await pending;
const call = context.fs.writeFile.mock.calls.find(([p]) => p.endsWith(".cursor.json"));
expect(call).toBeDefined();
expect(JSON.parse(call![1]).samples.map((s: { timeMs: number }) => s.timeMs)).toEqual([
100, 300, 500,
]);
expect(stoppedBeforeHelper).toBe(1);
});
it("bounds interrupted takes and closes a pause still active at stop", async () => {
const { context, finish, stop, result } = fixture();
result.interrupted = true;
result.cursorPauseRanges = [{ startMs: 200, endMs: 1500 }];
result.durationMs = 200;
const pending = stop({});
finish();
await pending;
const call = context.fs.writeFile.mock.calls.find(([p]) => p.endsWith(".cursor.json"));
expect(JSON.parse(call![1]).samples.map((s: { timeMs: number }) => s.timeMs)).toEqual([100]);
});
it.each([
"system",
"editable-overlay",
])("starts a sampler only for editable native mode (%s)", async (mode) => {
const nativeStart = source.indexOf('ipcMain.handle(\n\t\t"start-native-linux-recording"');
expect(nativeStart).toBeGreaterThan(-1);
const start = source.indexOf('if (cursorCaptureMode === "editable-overlay")', nativeStart);
const end = source.indexOf("\n\t\t\t\treturn { success: true, recordingId", start);
expect(end).toBeGreaterThan(start);
const startCursorRecording = vi.fn();
const context = { cursorCaptureMode: mode, startCursorRecording, recordingId: 7 };
const code = `(async () => {${source.slice(start, end)}})()`;
await runInNewContext(
ts.transpileModule(code, { compilerOptions: { target: ts.ScriptTarget.ES2022 } }).outputText,
context,
);
expect(startCursorRecording).toHaveBeenCalledTimes(mode === "editable-overlay" ? 1 : 0);
});

it("prefers helper telemetry without re-compacting it", async () => {
const { context, finish, stop } = fixture("editable-overlay", true);
const pending = stop({});
finish();
await pending;
const call = context.fs.writeFile.mock.calls.find(([p]) => p.endsWith(".cursor.json"));
expect(JSON.parse(call![1]).samples.map((s: { timeMs: number }) => s.timeMs)).toEqual([42]);
});
it("system cursor publishes no editable sidecar", async () => {
const { context, finish, stop } = fixture("system");
const pending = stop({});
finish();
await pending;
expect(context.fs.writeFile.mock.calls.some(([p]) => p.endsWith(".cursor.json"))).toBe(false);
});
it("discard publishes no sidecar and does not stop the helper for storage", async () => {
const { context, session, stop } = fixture();
await stop({}, true);
expect(session.discard).toHaveBeenCalledOnce();
expect(session.stop).not.toHaveBeenCalled();
expect(context.fs.writeFile).not.toHaveBeenCalled();
});
});
101 changes: 88 additions & 13 deletions electron/ipc/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -111,14 +111,22 @@ import {
type LinuxCaptureSourceKind,
LinuxNativeCaptureSession,
} from "../native-bridge/capture/linuxNativeCaptureSession";
import { compactCursorTelemetryPauseRanges } from "../native-bridge/cursor/recording/cursorPauseRanges";
import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory";
import {
compactCursorTelemetryPauseRanges,
finalizeSampledCursorTelemetry,
} from "../native-bridge/cursor/recording/cursorPauseRanges";
import {
createCursorRecordingSession,
createX11PointerRecordingSession,
isX11PointerFallbackEnabled,
} from "../native-bridge/cursor/recording/factory";
import {
isMacCursorHelperUnavailable,
requestMacCursorAccessibilityAccess,
} from "../native-bridge/cursor/recording/macNativeCursorRecordingSession";
import { findLinuxCaptureHelperPath } from "../native-bridge/cursor/recording/pipeWireCursorRecordingSession";
import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session";
import { resolveX11CaptureBounds } from "../native-bridge/cursor/recording/x11CaptureBounds";
import { toHelperRect } from "../native-bridge/helperCoordinates";
import {
isMacPickerSourceId,
Expand Down Expand Up @@ -1179,6 +1187,19 @@ function isMacWindowCaptureSourceId(sourceId: string | null | undefined) {
);
}

// Electron pointer positions and display bounds are DIP; helper origins may
// instead be X11 pixels. Resolve conservatively, never mix those coordinate spaces.
// Only native Linux capture runs this sampler, so the portal grant is the source of truth.
function getX11CaptureBounds() {
if (!linuxNativeCaptureSession) return null;
return resolveX11CaptureBounds({
displays: screen.getAllDisplays(),
sourceKind: linuxNativeCaptureSession.grantedSourceKind,
origin: linuxNativeCaptureSession.grantedCaptureOrigin,
size: linuxNativeCaptureSession.grantedCaptureSize,
});
}

function getSelectedSourceBounds() {
// A window take is normalised against the rect the helper is recording. That
// rect is seeded from the source frame before the first cursor sample and
Expand Down Expand Up @@ -1376,11 +1397,40 @@ async function resolveDirectShowWebcamClsid(deviceName?: string) {
async function startCursorRecording(recordingId?: number) {
// On Linux the native capture helper already produces cursor samples, from
// the SAME portal session that produces the pixels. Spawning the cursor-only
// helper alongside it would open a SECOND portal session — and since
// PipeWire helper alongside it would open a SECOND portal session — and since
// SelectSources may be called once per session, that means a second picker
// in front of the user, for a stream nothing consumes. The double prompt is
// precisely what the native path exists to remove.
//
// X11 is the exception: the portal often has no METADATA cursor mode at all,
// so the helper records pixels with an empty cursor track. Sampling the
// pointer through X11 (Electron / xinput) does not talk to the portal,
// so it can run in parallel and fill `<video>.cursor.json` when native samples
// come back empty.
if (linuxNativeCaptureSession) {
if (!isX11PointerFallbackEnabled()) {
return;
}
if (cursorRecordingSession) {
pendingCursorRecordingData = await cursorRecordingSession.stop();
cursorRecordingSession = null;
}
pendingCursorRecordingData = null;
// Use the helper's capture-started epoch, not prepare/picker time or the
// slightly later sampler start: pause ranges use that same epoch.
cursorRecordingSession = createX11PointerRecordingSession({
getDisplayBounds: getX11CaptureBounds,
maxSamples: MAX_CURSOR_SAMPLES,
sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS,
startTimeMs: linuxNativeCaptureSession.cursorStartTimeMs,
});
try {
await cursorRecordingSession.start();
console.info("[cursor-x11] sampling the pointer alongside native Linux capture");
} catch (error) {
console.error("Failed to start the X11 pointer sampler:", error);
cursorRecordingSession = null;
}
return;
}

Expand Down Expand Up @@ -2957,6 +3007,10 @@ export function registerIpcHandlers(
linuxNativeCaptureSourceLabel = linuxSourceLabel(session.grantedSourceKind);
setAppRecording(true, linuxNativeCaptureSourceLabel, recordingId);

if (cursorCaptureMode === "editable-overlay") {
await startCursorRecording(recordingId);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

return { success: true, recordingId, path: outputPath };
} catch (error) {
console.error("Failed to start native Linux recording:", error);
Expand Down Expand Up @@ -3003,6 +3057,8 @@ export function registerIpcHandlers(

try {
if (discard) {
await stopCursorRecording();
pendingCursorRecordingData = null;
session.discard();
const discarded = path.join(RECORDINGS_DIR, `${RECORDING_FILE_PREFIX}${recordingId}.mp4`);
await Promise.all([
Expand All @@ -3014,7 +3070,11 @@ export function registerIpcHandlers(

let result: Awaited<ReturnType<LinuxNativeCaptureSession["stop"]>>;
try {
result = await session.stop();
// Stop sampling at the stop request, not after helper teardown.
[result] = await Promise.all([
session.stop(),
cursorCaptureMode === "editable-overlay" ? stopCursorRecording() : Promise.resolve(),
]);
} catch (error) {
// A Stop the user asked for fails as before, with the helper's reason.
if (!session.interrupted) {
Expand All @@ -3040,15 +3100,28 @@ export function registerIpcHandlers(
? { screenVideoPath: result.path, message: result.warning }
: null;

// The helper collects cursor samples itself, from the same portal
// session that produced the pixels, so there is no separate sampler
// to stop and no clock offset to correct — the two are one recording.
if (cursorCaptureMode === "editable-overlay" && result.cursor.samples.length > 0) {
await fs.writeFile(
`${result.path}.cursor.json`,
JSON.stringify(result.cursor, null, 2),
"utf-8",
);
if (cursorCaptureMode !== "editable-overlay") pendingCursorRecordingData = null;

// Prefer the helper's own samples — they share the portal session that
// produced the pixels, so there is no clock offset to correct. When
// the portal had no METADATA cursor mode the track is empty; the X11
// sampler started alongside the take then fills the sidecar so
// `--auto-zoom` still has clicks to work from. Those fallback samples
// still include paused time, which the video does not. Apply the same
// compaction and duration bound as browser video (also after interruption).
const cursor =
result.cursor.samples.length > 0
? result.cursor
: pendingCursorRecordingData
? finalizeSampledCursorTelemetry(
pendingCursorRecordingData,
result.cursorPauseRanges,
result.durationMs,
)
: null;
pendingCursorRecordingData = null;
if (cursorCaptureMode === "editable-overlay" && cursor && cursor.samples.length > 0) {
await fs.writeFile(`${result.path}.cursor.json`, JSON.stringify(cursor, null, 2), "utf-8");
}

const session_: RecordingSession = {
Expand Down Expand Up @@ -3086,6 +3159,8 @@ export function registerIpcHandlers(
console.error("Failed to stop native Linux recording:", error);
return { success: false, error: String(error) };
} finally {
await stopCursorRecording();
pendingCursorRecordingData = null;
linuxNativeCaptureSession = null;
linuxNativeCaptureRecordingId = null;
linuxNativeCaptureCursorMode = "editable-overlay";
Expand Down
Loading
Loading