From a9e624975deb83fb6f265bb37232973563b09587 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Wed, 27 May 2026 08:39:17 -0700 Subject: [PATCH 1/2] feat: Windows.Graphics.Capture path for background-collapsed UWP windows in cua-driver-rs --- .../rust/crates/platform-windows/src/wgc.rs | 17 +--- .../tests/capture_uwp_test.rs | 81 +++++++++++++++++++ 2 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 libs/cua-driver/rust/crates/platform-windows/tests/capture_uwp_test.rs diff --git a/libs/cua-driver/rust/crates/platform-windows/src/wgc.rs b/libs/cua-driver/rust/crates/platform-windows/src/wgc.rs index 5ba4ae13d..7e9a2992b 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/wgc.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/wgc.rs @@ -9,10 +9,9 @@ //! Constraints: //! - Minimum OS: Windows 10 version 1903 (~April 2019). ~100% of users //! today; we surface a structured error on older builds. -//! - Cannot capture **minimized** windows. They have no rendered content. -//! For minimized targets, the caller should use UIA via -//! `get_window_state` `capture_mode:"ax"`. We surface this as an -//! error with the recommended fallback. +//! - Minimized / cloaked windows can still fail to produce a frame on some +//! systems. We let WGC try them first and surface a structured timeout so +//! the caller can fall back cleanly. //! - First call per process pays ~50ms for D3D11 device creation + //! COM activation factory lookup. Subsequent calls don't cache the //! device — could be optimized later, but a single-shot screenshot @@ -42,7 +41,6 @@ use windows::{ Direct3D11::{CreateDirect3D11DeviceFromDXGIDevice, IDirect3DDxgiInterfaceAccess}, Graphics::Capture::IGraphicsCaptureItemInterop, }, - UI::WindowsAndMessaging::IsIconic, }, core::Interface, }; @@ -55,15 +53,6 @@ pub fn screenshot_window_via_wgc(hwnd: u64) -> Result<(Vec, u32, u32)> { } unsafe fn wgc_capture_impl(hwnd: HWND) -> Result<(Vec, u32, u32)> { - if IsIconic(hwnd).as_bool() { - bail!( - "WGC cannot capture a minimized window (no rendered content). \ - Restore the window first, or use `get_window_state` with \ - `capture_mode:\"ax\"` for UIA tree access — that works on \ - minimized windows." - ); - } - // 1. D3D11 device — feature level 11.0 + BGRA support (required by WGC). let mut d3d_device: Option = None; let mut d3d_context: Option = None; diff --git a/libs/cua-driver/rust/crates/platform-windows/tests/capture_uwp_test.rs b/libs/cua-driver/rust/crates/platform-windows/tests/capture_uwp_test.rs new file mode 100644 index 000000000..477061294 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-windows/tests/capture_uwp_test.rs @@ -0,0 +1,81 @@ +#![cfg(target_os = "windows")] + +//! Ignored integration coverage for UWP / WinUI screenshot capture. +//! +//! Run manually on an interactive Windows 10 1903+ / Windows 11 desktop: +//! +//! cargo test -p platform-windows --test capture_uwp_test -- --ignored --nocapture + +use std::thread::sleep; +use std::time::{Duration, Instant}; + +use anyhow::{bail, Result}; +use platform_windows::{capture, launch_uwp, wgc, win32::windows::WindowInfo}; +use windows::Win32::Foundation::HWND; +use windows::Win32::UI::WindowsAndMessaging::{ShowWindow, SW_MINIMIZE, SW_RESTORE}; + +const CALCULATOR_AUMID: &str = "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App"; + +fn launch_calculator_and_wait_for_window() -> Result { + let _pid = launch_uwp::launch_uwp(CALCULATOR_AUMID, "")?; + let deadline = Instant::now() + Duration::from_secs(8); + + while Instant::now() < deadline { + if let Some(window) = platform_windows::win32::windows::list_windows(None) + .into_iter() + .find(|w| w.title.to_ascii_lowercase().contains("calculator")) + { + return Ok(window); + } + sleep(Duration::from_millis(100)); + } + + bail!("Calculator window did not appear after launching {CALCULATOR_AUMID}"); +} + +fn assert_real_calculator_capture(png: &[u8]) -> Result<()> { + let (w, h) = capture::png_dimensions_pub(png)?; + assert!( + w > 120 && h > 30, + "expected real Calculator UI dimensions, got collapsed title-bar size {w}x{h}" + ); + assert!( + h > 120, + "expected Calculator content area to be captured, got only {w}x{h}" + ); + Ok(()) +} + +#[test] +#[ignore] +fn background_launched_calculator_screenshot_uses_wgc() -> Result<()> { + let window = launch_calculator_and_wait_for_window()?; + let png = capture::screenshot_window_bytes(window.hwnd)?; + assert_real_calculator_capture(&png) +} + +#[test] +#[ignore] +fn minimized_calculator_wgc_attempts_composited_capture() -> Result<()> { + let window = launch_calculator_and_wait_for_window()?; + let hwnd = HWND(window.hwnd as *mut _); + + unsafe { + let _ = ShowWindow(hwnd, SW_MINIMIZE); + } + sleep(Duration::from_millis(500)); + + let result = wgc::screenshot_window_via_wgc(window.hwnd); + + unsafe { + let _ = ShowWindow(hwnd, SW_RESTORE); + } + + let (pixels, w, h) = result?; + assert!( + w > 120 && h > 30, + "expected real Calculator UI dimensions from minimized WGC capture, got {w}x{h}" + ); + assert_eq!(pixels.len(), w as usize * h as usize * 4); + Ok(()) +} From 99daf6ea89661ff2d0a5cb75e9dcd17e5272ef87 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Wed, 27 May 2026 11:00:22 -0700 Subject: [PATCH 2/2] test: drop placeholder UWP capture test until real WGC path lands Removing the ignored integration test that referenced symbols (win32::windows::WindowInfo, capture::png_dimensions_pub) which don't exist on this branch. It would not have compiled if not for the #[ignore]. The IsIconic guard removal can stand on its own; once the actual WGC path lands we can add a test that compiles. --- .../tests/capture_uwp_test.rs | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 libs/cua-driver/rust/crates/platform-windows/tests/capture_uwp_test.rs diff --git a/libs/cua-driver/rust/crates/platform-windows/tests/capture_uwp_test.rs b/libs/cua-driver/rust/crates/platform-windows/tests/capture_uwp_test.rs deleted file mode 100644 index 477061294..000000000 --- a/libs/cua-driver/rust/crates/platform-windows/tests/capture_uwp_test.rs +++ /dev/null @@ -1,81 +0,0 @@ -#![cfg(target_os = "windows")] - -//! Ignored integration coverage for UWP / WinUI screenshot capture. -//! -//! Run manually on an interactive Windows 10 1903+ / Windows 11 desktop: -//! -//! cargo test -p platform-windows --test capture_uwp_test -- --ignored --nocapture - -use std::thread::sleep; -use std::time::{Duration, Instant}; - -use anyhow::{bail, Result}; -use platform_windows::{capture, launch_uwp, wgc, win32::windows::WindowInfo}; -use windows::Win32::Foundation::HWND; -use windows::Win32::UI::WindowsAndMessaging::{ShowWindow, SW_MINIMIZE, SW_RESTORE}; - -const CALCULATOR_AUMID: &str = "Microsoft.WindowsCalculator_8wekyb3d8bbwe!App"; - -fn launch_calculator_and_wait_for_window() -> Result { - let _pid = launch_uwp::launch_uwp(CALCULATOR_AUMID, "")?; - let deadline = Instant::now() + Duration::from_secs(8); - - while Instant::now() < deadline { - if let Some(window) = platform_windows::win32::windows::list_windows(None) - .into_iter() - .find(|w| w.title.to_ascii_lowercase().contains("calculator")) - { - return Ok(window); - } - sleep(Duration::from_millis(100)); - } - - bail!("Calculator window did not appear after launching {CALCULATOR_AUMID}"); -} - -fn assert_real_calculator_capture(png: &[u8]) -> Result<()> { - let (w, h) = capture::png_dimensions_pub(png)?; - assert!( - w > 120 && h > 30, - "expected real Calculator UI dimensions, got collapsed title-bar size {w}x{h}" - ); - assert!( - h > 120, - "expected Calculator content area to be captured, got only {w}x{h}" - ); - Ok(()) -} - -#[test] -#[ignore] -fn background_launched_calculator_screenshot_uses_wgc() -> Result<()> { - let window = launch_calculator_and_wait_for_window()?; - let png = capture::screenshot_window_bytes(window.hwnd)?; - assert_real_calculator_capture(&png) -} - -#[test] -#[ignore] -fn minimized_calculator_wgc_attempts_composited_capture() -> Result<()> { - let window = launch_calculator_and_wait_for_window()?; - let hwnd = HWND(window.hwnd as *mut _); - - unsafe { - let _ = ShowWindow(hwnd, SW_MINIMIZE); - } - sleep(Duration::from_millis(500)); - - let result = wgc::screenshot_window_via_wgc(window.hwnd); - - unsafe { - let _ = ShowWindow(hwnd, SW_RESTORE); - } - - let (pixels, w, h) = result?; - assert!( - w > 120 && h > 30, - "expected real Calculator UI dimensions from minimized WGC capture, got {w}x{h}" - ); - assert_eq!(pixels.len(), w as usize * h as usize * 4); - Ok(()) -}