diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs index 53298d665..e633191f0 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs @@ -6,7 +6,7 @@ #![allow(non_upper_case_globals, non_camel_case_types, non_snake_case, dead_code)] use core_foundation::{ - array::CFArrayRef, + array::{CFArrayRef, CFIndex}, base::{CFRelease, CFRetain, CFTypeID, CFTypeRef}, string::CFStringRef, }; @@ -68,6 +68,39 @@ extern "C" { value: CFTypeRef, ) -> AXError; pub fn AXUIElementGetTypeID() -> CFTypeID; + + /// Fetch several attributes of an element in a single IPC round-trip. + /// `attributes` is a CFArray of attribute-name CFStrings; `values` is set + /// to a parallel CFArray with one slot per requested attribute (same order). + /// With `options = 0` a per-attribute failure is reported as an AXValue of + /// error type in that slot rather than aborting the whole batch; passing + /// `0x1` (StopOnError) would instead bail on the first failing attribute. + pub fn AXUIElementCopyMultipleAttributeValues( + element: AXUIElementRef, + attributes: CFArrayRef, + options: u32, + values: *mut CFArrayRef, + ) -> AXError; + + /// Fetch a contiguous range (`index`..`index+maxValues`) of an + /// array-valued attribute in one IPC round-trip. Used to cap how many + /// children of a single node we materialize. + pub fn AXUIElementCopyAttributeValues( + element: AXUIElementRef, + attribute: CFStringRef, + index: CFIndex, + maxValues: CFIndex, + values: *mut CFArrayRef, + ) -> AXError; + + /// Total element count of an array-valued attribute, without fetching it. + pub fn AXUIElementGetAttributeValueCount( + element: AXUIElementRef, + attribute: CFStringRef, + count: *mut CFIndex, + ) -> AXError; + + pub fn AXValueGetTypeID() -> CFTypeID; pub fn AXIsProcessTrusted() -> bool; /// `AXIsProcessTrustedWithOptions(options)` — when called with /// `{kAXTrustedCheckOptionPrompt: true}` raises the system Accessibility @@ -129,6 +162,130 @@ pub unsafe fn copy_action_names(element: AXUIElementRef) -> Vec { .collect() } +/// Fetch several string attributes of an element in ONE IPC round-trip. +/// +/// Returns one `Option` per requested attribute, in the same order as +/// `attr_names`: +/// - `Some(s)` when the slot came back as a CFString (including the empty +/// string `""` — an attribute that is *present but empty* stays distinct from +/// an *absent* attribute, which is critical for the AXTitle-vs-AXDescription +/// None-vs-empty semantics the tree formatter relies on); +/// - `None` when the attribute is unsupported/has no value (the slot is an +/// AXValue error placeholder) or is present but not a string. +/// +/// `options` is hard-coded to `0` (NOT StopOnError): one failing attribute must +/// not abort the batch — its slot is simply filtered out by position. +pub unsafe fn copy_multiple_attrs( + element: AXUIElementRef, + attr_names: &[&str], +) -> Vec> { + if attr_names.is_empty() { + return Vec::new(); + } + + // Build the CFArray of attribute-name CFStrings. + let cf_names: Vec = attr_names.iter().map(|n| CFStr::new(n)).collect(); + let names_array = CFArray::from_CFTypes(&cf_names); + + let mut values: CFArrayRef = std::ptr::null(); + let err = AXUIElementCopyMultipleAttributeValues( + element, + names_array.as_concrete_TypeRef(), + 0, // options = 0: do NOT stop on the first failing attribute. + &mut values, + ); + if err != kAXErrorSuccess || values.is_null() { + // Whole-batch failure — report every slot as absent. + return vec![None; attr_names.len()]; + } + + let arr = CFArray::::wrap_under_create_rule(values as _); + let cf_string_type_id = CFStr::type_id(); + let len = arr.len() as usize; + + (0..attr_names.len()) + .map(|i| { + if i >= len { + return None; + } + let item = match arr.get(i as CFIndex) { + Some(it) => *it, + None => return None, + }; + if item.is_null() { + return None; + } + // Per-slot failures come back as an AXValue of error type, not a + // CFString — filter them out by type so they read as "absent". + if core_foundation::base::CFGetTypeID(item) != cf_string_type_id { + return None; + } + // Borrow (get-rule): the array owns the element; clone into a String. + let s = CFStr::wrap_under_get_rule(item as _); + Some(s.to_string()) + }) + .collect() +} + +/// Fetch up to `max` children of an element in ONE IPC round-trip via the +/// ranged-attribute API, instead of pulling the entire (possibly huge) +/// AXChildren array. Returns a Vec of retained AXUIElementRefs the caller must +/// release, plus a flag that is `true` when the node has MORE children than +/// were returned (i.e. the list was clipped at `max`). +pub unsafe fn copy_children_ranged( + element: AXUIElementRef, + max: CFIndex, +) -> (Vec, bool) { + let attr = CFStr::new("AXChildren"); + + // How many children does this node actually have? Used only to decide + // whether the ranged fetch clipped anything. + let mut total: CFIndex = 0; + let count_err = AXUIElementGetAttributeValueCount( + element, + attr.as_concrete_TypeRef(), + &mut total, + ); + let total_known = count_err == kAXErrorSuccess; + + let mut value: CFArrayRef = std::ptr::null(); + let err = AXUIElementCopyAttributeValues( + element, + attr.as_concrete_TypeRef(), + 0, + max, + &mut value, + ); + if err != kAXErrorSuccess || value.is_null() { + return (vec![], false); + } + + let arr = CFArray::::wrap_under_create_rule(value as _); + let ax_type_id = AXUIElementGetTypeID(); + let children: Vec = (0..arr.len()) + .filter_map(|i| { + let item = *arr.get(i)?; + if core_foundation::base::CFGetTypeID(item) == ax_type_id { + // Retain so we own it — caller is responsible for releasing. + CFRetain(item); + Some(item as AXUIElementRef) + } else { + None + } + }) + .collect(); + + // Clipped when the node reports more children than we fetched. If the count + // call failed we fall back to "fetched exactly max" as the clip heuristic. + let clipped = if total_known { + total > max + } else { + arr.len() >= max + }; + + (children, clipped) +} + /// Read the on-screen center of an AX element (AXPosition + AXSize → center). /// Returns `(cx, cy)` in screen coordinates, or `None` if either attribute /// is unavailable or the element has zero size. diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs index a05f0924c..fd272e632 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs @@ -25,6 +25,14 @@ const MAX_DEPTH: usize = 25; /// with a warning line appended (mirrors Swift reference implementation). const MAX_ELEMENTS: usize = 2_000; +/// Per-node cap on how many children we fetch via the ranged-children API. +/// Distinct from MAX_ELEMENTS (a total-walk cap): this bounds a single +/// pathological node (e.g. a virtualized list/grid exposing thousands of +/// rows) so one node can't dominate the walk, while the total cap bounds the +/// walk as a whole. When a node's children are clipped here a per-node note +/// is appended to the tree. +const MAX_CHILDREN: core_foundation::array::CFIndex = 200; + /// A single node in the AX tree. #[derive(Debug, Clone)] pub struct AXNode { @@ -165,17 +173,33 @@ unsafe fn walk_element( } *visited_count += 1; - let role = copy_string_attr(element, "AXRole") - .unwrap_or_else(|| "AXUnknown".into()); + // Batch every string attribute we care about into ONE IPC round-trip + // instead of ~7 separate AXUIElementCopyAttributeValue calls. Order is + // load-bearing — each slot is indexed by position below. + const ATTRS: [&str; 7] = [ + "AXRole", // 0 + "AXTitle", // 1 + "AXValue", // 2 + "AXPlaceholderValue",// 3 — fallback for empty text fields + "AXDescription", // 4 + "AXIdentifier", // 5 + "AXHelp", // 6 + ]; + let attrs = copy_multiple_attrs(element, &ATTRS); + + let role = attrs[0].clone().unwrap_or_else(|| "AXUnknown".into()); // Skip pure layout containers that have no interesting content. if role == "AXScrollArea" || role == "AXGroup" { // Still recurse — children may be interesting. - let children = copy_children(element); + let (children, child_clipped) = copy_children_ranged(element, MAX_CHILDREN); for child in children { walk_element(child, depth, nodes, lines, counter, visited_count, truncated); CFRelease(child as CFTypeRef); } + if child_clipped { + note_clipped_children(lines, depth); + } return; } @@ -184,14 +208,22 @@ unsafe fn walk_element( // This is critical for Calculator where AXTitle="" but AXDescription="2" // (digit buttons). Merging them would produce "2" (quoted) instead of (2) // (parens), breaking _find_calc_button which searches for "(2)". - let title = copy_string_attr(element, "AXTitle"); - let value = copy_string_attr(element, "AXValue"); - // AXPlaceholderValue as fallback for empty text fields. - let value = value.filter(|v| !v.trim().is_empty()) - .or_else(|| copy_string_attr(element, "AXPlaceholderValue")); - let description = copy_string_attr(element, "AXDescription"); - let identifier = copy_string_attr(element, "AXIdentifier"); - let help = copy_string_attr(element, "AXHelp").filter(|h| !h.trim().is_empty()); + // + // None-vs-empty matters here: an attribute that is present-but-empty ("") + // must stay distinct from an absent attribute. copy_multiple_attrs preserves + // that — a present empty slot is Some("") and an unsupported slot is None — + // so the AXTitle="" / AXDescription="2" Calculator case is unaffected by + // batching. + let title = attrs[1].clone(); + // AXValue, with AXPlaceholderValue as fallback when AXValue is missing/blank. + let value = attrs[2].clone() + .filter(|v| !v.trim().is_empty()) + .or_else(|| attrs[3].clone()); + let description = attrs[4].clone(); + let identifier = attrs[5].clone(); + let help = attrs[6].clone().filter(|h| !h.trim().is_empty()); + // Action names remain a separate call — they come from a different AX API + // (AXUIElementCopyActionNames) and can't ride the attribute batch. let actions = copy_action_names(element); let visible_title = title.as_deref().unwrap_or("").trim().to_owned(); @@ -204,11 +236,14 @@ unsafe fn walk_element( let is_actionable = !actions.is_empty(); if !is_actionable && !has_content && role != "AXWindow" && role != "AXSheet" { - let children = copy_children(element); + let (children, child_clipped) = copy_children_ranged(element, MAX_CHILDREN); for child in children { walk_element(child, depth + 1, nodes, lines, counter, visited_count, truncated); CFRelease(child as CFTypeRef); } + if child_clipped { + note_clipped_children(lines, depth + 1); + } return; } @@ -248,11 +283,25 @@ unsafe fn walk_element( lines.push((depth, line)); nodes.push(node); - let children = copy_children(element); + let (children, child_clipped) = copy_children_ranged(element, MAX_CHILDREN); for child in children { walk_element(child, depth + 1, nodes, lines, counter, visited_count, truncated); CFRelease(child as CFTypeRef); } + if child_clipped { + note_clipped_children(lines, depth + 1); + } +} + +/// Append a per-node truncation note when a node's children were clipped at the +/// per-node `MAX_CHILDREN` cap. This is a *different concern* from the total +/// MAX_ELEMENTS node cap: a single node with thousands of children (a giant +/// list/grid) is capped locally without aborting the whole walk. +fn note_clipped_children(lines: &mut Vec<(usize, String)>, depth: usize) { + lines.push(( + depth, + format!("- … (children clipped at {MAX_CHILDREN})"), + )); } fn format_node_line(node: &AXNode) -> String {