From 939768794966c0cb799ebc2c1cae50abe4eb42b8 Mon Sep 17 00:00:00 2001 From: "Dhanji R. Prasanna" Date: Wed, 21 Jan 2026 06:54:31 +0530 Subject: [PATCH] Remove unused mouse control and macax accessibility code Removed dead code that was never used by any g3 tool: - macax/ module (accessibility control via AXApplication, AXElement) - move_mouse() and click_at() methods from ComputerController trait - macax_demo.rs and test_type_text.rs examples The ComputerController trait now only has take_screenshot(), which is the only method actually used by the screenshot tool. --- .../examples/macax_demo.rs | 74 -- .../examples/test_type_text.rs | 48 -- crates/g3-computer-control/src/lib.rs | 8 - .../src/macax/controller.rs | 808 ------------------ crates/g3-computer-control/src/macax/mod.rs | 65 -- crates/g3-computer-control/src/macax/tests.rs | 37 - .../g3-computer-control/src/platform/linux.rs | 8 - .../g3-computer-control/src/platform/macos.rs | 93 +- .../src/platform/windows.rs | 8 - 9 files changed, 1 insertion(+), 1148 deletions(-) delete mode 100644 crates/g3-computer-control/examples/macax_demo.rs delete mode 100644 crates/g3-computer-control/examples/test_type_text.rs delete mode 100644 crates/g3-computer-control/src/macax/controller.rs delete mode 100644 crates/g3-computer-control/src/macax/mod.rs delete mode 100644 crates/g3-computer-control/src/macax/tests.rs diff --git a/crates/g3-computer-control/examples/macax_demo.rs b/crates/g3-computer-control/examples/macax_demo.rs deleted file mode 100644 index 1eefe34..0000000 --- a/crates/g3-computer-control/examples/macax_demo.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Example demonstrating macOS Accessibility API tools -//! -//! This example shows how to use the macax tools to control macOS applications. -//! -//! Run with: cargo run --example macax_demo - -use anyhow::Result; -use g3_computer_control::MacAxController; - -#[tokio::main] -async fn main() -> Result<()> { - println!("🍎 macOS Accessibility API Demo\n"); - println!("This demo shows how to control macOS applications using the Accessibility API.\n"); - - // Create controller - let controller = MacAxController::new()?; - println!("βœ… MacAxController initialized\n"); - - // List running applications - println!("πŸ“± Listing running applications:"); - match controller.list_applications() { - Ok(apps) => { - for app in apps.iter().take(10) { - println!(" - {}", app.name); - } - if apps.len() > 10 { - println!(" ... and {} more", apps.len() - 10); - } - } - Err(e) => println!(" ❌ Error: {}", e), - } - println!(); - - // Get frontmost app - println!("🎯 Getting frontmost application:"); - match controller.get_frontmost_app() { - Ok(app) => println!(" Current: {}", app.name), - Err(e) => println!(" ❌ Error: {}", e), - } - println!(); - - // Example: Activate Finder and get its UI tree - println!("πŸ“‚ Activating Finder and inspecting UI:"); - match controller.activate_app("Finder") { - Ok(_) => { - println!(" βœ… Finder activated"); - - // Wait a moment for activation - tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; - - // Get UI tree - match controller.get_ui_tree("Finder", 2) { - Ok(tree) => { - println!("\n UI Tree:"); - for line in tree.lines().take(10) { - println!(" {}", line); - } - } - Err(e) => println!(" ❌ Error getting UI tree: {}", e), - } - } - Err(e) => println!(" ❌ Error: {}", e), - } - println!(); - - println!("✨ Demo complete!\n"); - println!("πŸ’‘ Tips:"); - println!(" - Use --macax flag with g3 to enable these tools"); - println!(" - Grant accessibility permissions in System Preferences"); - println!(" - Add accessibility identifiers to your apps for easier automation"); - println!(" - See docs/macax-tools.md for full documentation\n"); - - Ok(()) -} diff --git a/crates/g3-computer-control/examples/test_type_text.rs b/crates/g3-computer-control/examples/test_type_text.rs deleted file mode 100644 index eb5d416..0000000 --- a/crates/g3-computer-control/examples/test_type_text.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! Test the new type_text functionality - -use anyhow::Result; -use g3_computer_control::MacAxController; - -#[tokio::main] -async fn main() -> Result<()> { - println!("πŸ§ͺ Testing macax type_text functionality\n"); - - let controller = MacAxController::new()?; - println!("βœ… Controller initialized\n"); - - // Test 1: Type simple text - println!("Test 1: Typing simple text into TextEdit"); - println!(" Please open TextEdit and create a new document..."); - std::thread::sleep(std::time::Duration::from_secs(3)); - - match controller.type_text("TextEdit", "Hello, World!") { - Ok(_) => println!(" βœ… Successfully typed simple text\n"), - Err(e) => println!(" ❌ Failed: {}\n", e), - } - - std::thread::sleep(std::time::Duration::from_secs(1)); - - // Test 2: Type unicode and emojis - println!("Test 2: Typing unicode and emojis"); - match controller.type_text("TextEdit", "\n🌟 Unicode test: cafΓ©, naΓ―ve, ζ—₯本θͺž πŸŽ‰") { - Ok(_) => println!(" βœ… Successfully typed unicode text\n"), - Err(e) => println!(" ❌ Failed: {}\n", e), - } - - std::thread::sleep(std::time::Duration::from_secs(1)); - - // Test 3: Type special characters - println!("Test 3: Typing special characters"); - match controller.type_text("TextEdit", "\nSpecial: @#$%^&*()_+-=[]{}|;':,.<>?/") { - Ok(_) => println!(" βœ… Successfully typed special characters\n"), - Err(e) => println!(" ❌ Failed: {}\n", e), - } - - println!("\n✨ Tests complete!"); - println!("\nπŸ’‘ Now try with Things3:"); - println!(" 1. Open Things3"); - println!(" 2. Press Cmd+N to create a new task"); - println!(" 3. Run: g3 --macax 'type \"🌟 My awesome task\" into Things'"); - - Ok(()) -} diff --git a/crates/g3-computer-control/src/lib.rs b/crates/g3-computer-control/src/lib.rs index 23632ea..8468188 100644 --- a/crates/g3-computer-control/src/lib.rs +++ b/crates/g3-computer-control/src/lib.rs @@ -1,7 +1,6 @@ // Suppress warnings from objc crate macros #![allow(unexpected_cfgs)] -pub mod macax; pub mod platform; pub mod types; pub mod webdriver; @@ -12,9 +11,6 @@ pub use webdriver::{ diagnostics::{run_diagnostics as run_chrome_diagnostics, ChromeDiagnosticReport, DiagnosticStatus}, }; -// Re-export macax types for convenience -pub use macax::{AXApplication, AXElement, MacAxController}; - use anyhow::Result; use async_trait::async_trait; use types::*; @@ -28,10 +24,6 @@ pub trait ComputerController: Send + Sync { region: Option, window_id: Option<&str>, ) -> Result<()>; - - // Mouse operations - fn move_mouse(&self, x: i32, y: i32) -> Result<()>; - fn click_at(&self, x: i32, y: i32, app_name: Option<&str>) -> Result<()>; } // Platform-specific constructor diff --git a/crates/g3-computer-control/src/macax/controller.rs b/crates/g3-computer-control/src/macax/controller.rs deleted file mode 100644 index aa3d529..0000000 --- a/crates/g3-computer-control/src/macax/controller.rs +++ /dev/null @@ -1,808 +0,0 @@ -use super::{AXApplication, AXElement}; -use anyhow::{Context, Result}; -use std::collections::HashMap; - -#[cfg(target_os = "macos")] -use accessibility::{ - AXUIElement, AXUIElementAttributes, ElementFinder, TreeVisitor, TreeWalker, TreeWalkerFlow, -}; - -#[cfg(target_os = "macos")] -use core_foundation::base::TCFType; - -#[cfg(target_os = "macos")] -use core_foundation::string::CFString; - -/// macOS Accessibility API controller using native APIs -pub struct MacAxController { - // Cache for application elements - app_cache: std::sync::Mutex>, -} - -impl MacAxController { - pub fn new() -> Result { - #[cfg(target_os = "macos")] - { - // Check if we have accessibility permissions by trying to get system-wide element - let _system = AXUIElement::system_wide(); - - Ok(Self { - app_cache: std::sync::Mutex::new(HashMap::new()), - }) - } - - #[cfg(not(target_os = "macos"))] - { - anyhow::bail!("macOS Accessibility API is only available on macOS") - } - } - - /// List all running applications - #[cfg(target_os = "macos")] - pub fn list_applications(&self) -> Result> { - let apps = Self::get_running_applications()?; - Ok(apps) - } - - #[cfg(not(target_os = "macos"))] - pub fn list_applications(&self) -> Result> { - anyhow::bail!("Not supported on this platform") - } - - #[cfg(target_os = "macos")] - fn get_running_applications() -> Result> { - use cocoa::appkit::NSApplicationActivationPolicy; - use cocoa::base::{id, nil}; - use objc::{class, msg_send, sel, sel_impl}; - - unsafe { - let workspace: id = msg_send![class!(NSWorkspace), sharedWorkspace]; - let running_apps: id = msg_send![workspace, runningApplications]; - let count: usize = msg_send![running_apps, count]; - - let mut apps = Vec::new(); - - for i in 0..count { - let app: id = msg_send![running_apps, objectAtIndex: i]; - - // Get app name - let localized_name: id = msg_send![app, localizedName]; - if localized_name == nil { - continue; - } - let name_ptr: *const i8 = msg_send![localized_name, UTF8String]; - let name = if !name_ptr.is_null() { - std::ffi::CStr::from_ptr(name_ptr) - .to_string_lossy() - .to_string() - } else { - continue; - }; - - // Get bundle ID - let bundle_id_obj: id = msg_send![app, bundleIdentifier]; - let bundle_id = if bundle_id_obj != nil { - let bundle_id_ptr: *const i8 = msg_send![bundle_id_obj, UTF8String]; - if !bundle_id_ptr.is_null() { - Some( - std::ffi::CStr::from_ptr(bundle_id_ptr) - .to_string_lossy() - .to_string(), - ) - } else { - None - } - } else { - None - }; - - // Get PID - let pid: i32 = msg_send![app, processIdentifier]; - - // Skip background-only apps - let activation_policy: i64 = msg_send![app, activationPolicy]; - if activation_policy - == NSApplicationActivationPolicy::NSApplicationActivationPolicyRegular as i64 - { - apps.push(AXApplication { - name, - bundle_id, - pid, - }); - } - } - - Ok(apps) - } - } - - /// Get the frontmost (active) application - #[cfg(target_os = "macos")] - pub fn get_frontmost_app(&self) -> Result { - use cocoa::base::{id, nil}; - use objc::{class, msg_send, sel, sel_impl}; - - unsafe { - let workspace: id = msg_send![class!(NSWorkspace), sharedWorkspace]; - let frontmost_app: id = msg_send![workspace, frontmostApplication]; - - if frontmost_app == nil { - anyhow::bail!("No frontmost application"); - } - - // Get app name - let localized_name: id = msg_send![frontmost_app, localizedName]; - let name_ptr: *const i8 = msg_send![localized_name, UTF8String]; - let name = std::ffi::CStr::from_ptr(name_ptr) - .to_string_lossy() - .to_string(); - - // Get bundle ID - let bundle_id_obj: id = msg_send![frontmost_app, bundleIdentifier]; - let bundle_id = if bundle_id_obj != nil { - let bundle_id_ptr: *const i8 = msg_send![bundle_id_obj, UTF8String]; - if !bundle_id_ptr.is_null() { - Some( - std::ffi::CStr::from_ptr(bundle_id_ptr) - .to_string_lossy() - .to_string(), - ) - } else { - None - } - } else { - None - }; - - // Get PID - let pid: i32 = msg_send![frontmost_app, processIdentifier]; - - Ok(AXApplication { - name, - bundle_id, - pid, - }) - } - } - - #[cfg(not(target_os = "macos"))] - pub fn get_frontmost_app(&self) -> Result { - anyhow::bail!("Not supported on this platform") - } - - /// Get AXUIElement for an application by name or PID - #[cfg(target_os = "macos")] - fn get_app_element(&self, app_name: &str) -> Result { - // Check cache first - { - let cache = self.app_cache.lock().unwrap(); - if let Some(element) = cache.get(app_name) { - return Ok(element.clone()); - } - } - - // Find the app by name - let apps = Self::get_running_applications()?; - let app = apps - .iter() - .find(|a| a.name == app_name) - .ok_or_else(|| anyhow::anyhow!("Application '{}' not found", app_name))?; - - // Create AXUIElement for the app - let element = AXUIElement::application(app.pid); - - // Cache it - { - let mut cache = self.app_cache.lock().unwrap(); - cache.insert(app_name.to_string(), element.clone()); - } - - Ok(element) - } - - /// Activate (bring to front) an application - #[cfg(target_os = "macos")] - pub fn activate_app(&self, app_name: &str) -> Result<()> { - use cocoa::base::id; - use objc::{class, msg_send, sel, sel_impl}; - - // Find the app - let apps = Self::get_running_applications()?; - let app = apps - .iter() - .find(|a| a.name == app_name) - .ok_or_else(|| anyhow::anyhow!("Application '{}' not found", app_name))?; - - unsafe { - let workspace: id = msg_send![class!(NSWorkspace), sharedWorkspace]; - let running_apps: id = msg_send![workspace, runningApplications]; - let count: usize = msg_send![running_apps, count]; - - for i in 0..count { - let running_app: id = msg_send![running_apps, objectAtIndex: i]; - let pid: i32 = msg_send![running_app, processIdentifier]; - - if pid == app.pid { - let _: bool = msg_send![running_app, activateWithOptions: 0]; - return Ok(()); - } - } - } - - anyhow::bail!("Failed to activate application") - } - - #[cfg(not(target_os = "macos"))] - pub fn activate_app(&self, _app_name: &str) -> Result<()> { - anyhow::bail!("Not supported on this platform") - } - - /// Get the UI hierarchy of an application - #[cfg(target_os = "macos")] - pub fn get_ui_tree(&self, app_name: &str, max_depth: usize) -> Result { - let app_element = self.get_app_element(app_name)?; - let mut output = format!("Application: {}\n", app_name); - - Self::build_ui_tree(&app_element, &mut output, 0, max_depth)?; - - Ok(output) - } - - #[cfg(not(target_os = "macos"))] - pub fn get_ui_tree(&self, _app_name: &str, _max_depth: usize) -> Result { - anyhow::bail!("Not supported on this platform") - } - - #[cfg(target_os = "macos")] - fn build_ui_tree( - element: &AXUIElement, - output: &mut String, - depth: usize, - max_depth: usize, - ) -> Result<()> { - if depth >= max_depth { - return Ok(()); - } - - let indent = " ".repeat(depth); - - // Get role - let role = element - .role() - .ok() - .map(|s| s.to_string()) - .unwrap_or_else(|| "Unknown".to_string()); - - // Get title - let title = element.title().ok().map(|s| s.to_string()); - - // Get identifier - let identifier = element.identifier().ok().map(|s| s.to_string()); - - // Format output - output.push_str(&format!("{}Role: {}", indent, role)); - if let Some(t) = title { - output.push_str(&format!(", Title: {}", t)); - } - if let Some(id) = identifier { - output.push_str(&format!(", ID: {}", id)); - } - output.push('\n'); - - // Get children - if let Ok(children) = element.children() { - for i in 0..children.len() { - if let Some(child) = children.get(i) { - let _ = Self::build_ui_tree(&child, output, depth + 1, max_depth); - } - } - } - - Ok(()) - } - - /// Find UI elements in an application - #[cfg(target_os = "macos")] - pub fn find_elements( - &self, - app_name: &str, - role: Option<&str>, - title: Option<&str>, - identifier: Option<&str>, - ) -> Result> { - let app_element = self.get_app_element(app_name)?; - let mut found_elements = Vec::new(); - - let visitor = ElementCollector { - role_filter: role.map(|s| s.to_string()), - title_filter: title.map(|s| s.to_string()), - identifier_filter: identifier.map(|s| s.to_string()), - results: std::cell::RefCell::new(&mut found_elements), - depth: std::cell::Cell::new(0), - }; - - let walker = TreeWalker::new(); - walker.walk(&app_element, &visitor); - - Ok(found_elements) - } - - #[cfg(not(target_os = "macos"))] - pub fn find_elements( - &self, - _app_name: &str, - _role: Option<&str>, - _title: Option<&str>, - _identifier: Option<&str>, - ) -> Result> { - anyhow::bail!("Not supported on this platform") - } - - /// Find a single element (helper for click, set_value, etc.) - #[cfg(target_os = "macos")] - fn find_element( - &self, - app_name: &str, - role: &str, - title: Option<&str>, - identifier: Option<&str>, - ) -> Result { - let app_element = self.get_app_element(app_name)?; - - let role_str = role.to_string(); - let title_str = title.map(|s| s.to_string()); - let identifier_str = identifier.map(|s| s.to_string()); - - let finder = ElementFinder::new( - &app_element, - move |element| { - // Check role - let elem_role = element.role().ok().map(|s| s.to_string()); - - if let Some(r) = elem_role { - if !r.contains(&role_str) { - return false; - } - } else { - return false; - } - - // Check title if specified - if let Some(ref title_filter) = title_str { - let elem_title = element.title().ok().map(|s| s.to_string()); - - if let Some(t) = elem_title { - if !t.contains(title_filter) { - return false; - } - } else { - return false; - } - } - - // Check identifier if specified - if let Some(ref id_filter) = identifier_str { - let elem_id = element.identifier().ok().map(|s| s.to_string()); - - if let Some(id) = elem_id { - if !id.contains(id_filter) { - return false; - } - } else { - return false; - } - } - - true - }, - Some(std::time::Duration::from_secs(2)), - ); - - finder.find().context("Element not found") - } - - /// Click on a UI element - #[cfg(target_os = "macos")] - pub fn click_element( - &self, - app_name: &str, - role: &str, - title: Option<&str>, - identifier: Option<&str>, - ) -> Result<()> { - let element = self.find_element(app_name, role, title, identifier)?; - - // Perform the press action - let action_name = CFString::new("AXPress"); - element - .perform_action(&action_name) - .map_err(|e| anyhow::anyhow!("Failed to perform press action: {:?}", e))?; - - Ok(()) - } - - #[cfg(not(target_os = "macos"))] - pub fn click_element( - &self, - _app_name: &str, - _role: &str, - _title: Option<&str>, - _identifier: Option<&str>, - ) -> Result<()> { - anyhow::bail!("Not supported on this platform") - } - - /// Set the value of a UI element - #[cfg(target_os = "macos")] - pub fn set_value( - &self, - app_name: &str, - role: &str, - value: &str, - title: Option<&str>, - identifier: Option<&str>, - ) -> Result<()> { - let element = self.find_element(app_name, role, title, identifier)?; - - // Set the value - convert CFString to CFType - let cf_value = CFString::new(value); - - element - .set_value(cf_value.as_CFType()) - .map_err(|e| anyhow::anyhow!("Failed to set value: {:?}", e))?; - - Ok(()) - } - - #[cfg(not(target_os = "macos"))] - pub fn set_value( - &self, - _app_name: &str, - _role: &str, - _value: &str, - _title: Option<&str>, - _identifier: Option<&str>, - ) -> Result<()> { - anyhow::bail!("Not supported on this platform") - } - - /// Get the value of a UI element - #[cfg(target_os = "macos")] - pub fn get_value( - &self, - app_name: &str, - role: &str, - title: Option<&str>, - identifier: Option<&str>, - ) -> Result { - let element = self.find_element(app_name, role, title, identifier)?; - - // Get the value - let value_type = element - .value() - .map_err(|e| anyhow::anyhow!("Failed to get value: {:?}", e))?; - - // Try to downcast to CFString - if let Some(cf_string) = value_type.downcast::() { - Ok(cf_string.to_string()) - } else { - // For non-string values, try to get a description - Ok(format!("")) - } - } - - #[cfg(not(target_os = "macos"))] - pub fn get_value( - &self, - _app_name: &str, - _role: &str, - _title: Option<&str>, - _identifier: Option<&str>, - ) -> Result { - anyhow::bail!("Not supported on this platform") - } - - /// Type text into the currently focused element (uses system text input) - #[cfg(target_os = "macos")] - pub fn type_text(&self, app_name: &str, text: &str) -> Result<()> { - use cocoa::base::{id, nil}; - use cocoa::foundation::NSString; - use objc::{class, msg_send, sel, sel_impl}; - - // First, make sure the app is active - self.activate_app(app_name)?; - - // Wait for app to fully activate - std::thread::sleep(std::time::Duration::from_millis(500)); - - // Send a Tab key to try to focus on a text field - // This helps ensure something is focused before we paste - let _ = self.press_key(app_name, "tab", vec![]); - std::thread::sleep(std::time::Duration::from_millis(800)); - - // Save old clipboard, set new content, paste, then restore - let old_content: id; - unsafe { - // Get the general pasteboard - let pasteboard: id = msg_send![class!(NSPasteboard), generalPasteboard]; - - // Save current clipboard content - let ns_string_type = NSString::alloc(nil).init_str("public.utf8-plain-text"); - old_content = msg_send![pasteboard, stringForType: ns_string_type]; - - // Clear and set new content - let _: () = msg_send![pasteboard, clearContents]; - - let ns_string = NSString::alloc(nil).init_str(text); - let ns_type = NSString::alloc(nil).init_str("public.utf8-plain-text"); - let _: bool = msg_send![pasteboard, setString:ns_string forType:ns_type]; - } - - // Wait a moment for clipboard to update - std::thread::sleep(std::time::Duration::from_millis(200)); - - // Paste using Cmd+V (outside unsafe block) - self.press_key(app_name, "v", vec!["command"])?; - - // Wait for paste to complete - std::thread::sleep(std::time::Duration::from_millis(300)); - - // Restore old clipboard content if it existed - unsafe { - if old_content != nil { - let pasteboard: id = msg_send![class!(NSPasteboard), generalPasteboard]; - let _: () = msg_send![pasteboard, clearContents]; - let ns_type = NSString::alloc(nil).init_str("public.utf8-plain-text"); - let _: bool = msg_send![pasteboard, setString:old_content forType:ns_type]; - } - } - - Ok(()) - } - - #[cfg(not(target_os = "macos"))] - pub fn type_text(&self, _app_name: &str, _text: &str) -> Result<()> { - anyhow::bail!("Not supported on this platform") - } - - /// Focus on a text field or text area element - #[cfg(target_os = "macos")] - pub fn focus_element( - &self, - app_name: &str, - role: &str, - title: Option<&str>, - identifier: Option<&str>, - ) -> Result<()> { - let element = self.find_element(app_name, role, title, identifier)?; - - // Set focused attribute to true - use core_foundation::boolean::CFBoolean; - let cf_true = CFBoolean::true_value(); - - element - .set_attribute(&accessibility::AXAttribute::focused(), cf_true) - .map_err(|e| anyhow::anyhow!("Failed to focus element: {:?}", e))?; - - Ok(()) - } - - /// Press a keyboard shortcut - #[cfg(target_os = "macos")] - pub fn press_key(&self, app_name: &str, key: &str, modifiers: Vec<&str>) -> Result<()> { - use core_graphics::event::{CGEvent, CGEventFlags, CGEventTapLocation}; - use core_graphics::event_source::{CGEventSource, CGEventSourceStateID}; - - // First, make sure the app is active - self.activate_app(app_name)?; - - // Wait a bit for activation - std::thread::sleep(std::time::Duration::from_millis(100)); - - // Map key string to key code - let key_code = - Self::key_to_keycode(key).ok_or_else(|| anyhow::anyhow!("Unknown key: {}", key))?; - - // Map modifiers to flags - let mut flags = CGEventFlags::CGEventFlagNull; - for modifier in modifiers { - match modifier.to_lowercase().as_str() { - "command" | "cmd" => flags |= CGEventFlags::CGEventFlagCommand, - "option" | "alt" => flags |= CGEventFlags::CGEventFlagAlternate, - "control" | "ctrl" => flags |= CGEventFlags::CGEventFlagControl, - "shift" => flags |= CGEventFlags::CGEventFlagShift, - _ => {} - } - } - - // Create event source - let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) - .ok() - .context("Failed to create event source")?; - - // Create key down event - let key_down = CGEvent::new_keyboard_event(source.clone(), key_code, true) - .ok() - .context("Failed to create key down event")?; - key_down.set_flags(flags); - - // Create key up event - let key_up = CGEvent::new_keyboard_event(source, key_code, false) - .ok() - .context("Failed to create key up event")?; - key_up.set_flags(flags); - - // Post events - key_down.post(CGEventTapLocation::HID); - std::thread::sleep(std::time::Duration::from_millis(50)); - key_up.post(CGEventTapLocation::HID); - - Ok(()) - } - - #[cfg(not(target_os = "macos"))] - pub fn press_key(&self, _app_name: &str, _key: &str, _modifiers: Vec<&str>) -> Result<()> { - anyhow::bail!("Not supported on this platform") - } - - #[cfg(target_os = "macos")] - fn key_to_keycode(key: &str) -> Option { - // Map common keys to keycodes - // See: https://eastmanreference.com/complete-list-of-applescript-key-codes - match key.to_lowercase().as_str() { - "a" => Some(0x00), - "s" => Some(0x01), - "d" => Some(0x02), - "f" => Some(0x03), - "h" => Some(0x04), - "g" => Some(0x05), - "z" => Some(0x06), - "x" => Some(0x07), - "c" => Some(0x08), - "v" => Some(0x09), - "b" => Some(0x0B), - "q" => Some(0x0C), - "w" => Some(0x0D), - "e" => Some(0x0E), - "r" => Some(0x0F), - "y" => Some(0x10), - "t" => Some(0x11), - "1" => Some(0x12), - "2" => Some(0x13), - "3" => Some(0x14), - "4" => Some(0x15), - "6" => Some(0x16), - "5" => Some(0x17), - "=" => Some(0x18), - "9" => Some(0x19), - "7" => Some(0x1A), - "-" => Some(0x1B), - "8" => Some(0x1C), - "0" => Some(0x1D), - "]" => Some(0x1E), - "o" => Some(0x1F), - "u" => Some(0x20), - "[" => Some(0x21), - "i" => Some(0x22), - "p" => Some(0x23), - "return" | "enter" => Some(0x24), - "l" => Some(0x25), - "j" => Some(0x26), - "'" => Some(0x27), - "k" => Some(0x28), - ";" => Some(0x29), - "\\" => Some(0x2A), - "," => Some(0x2B), - "/" => Some(0x2C), - "n" => Some(0x2D), - "m" => Some(0x2E), - "." => Some(0x2F), - "tab" => Some(0x30), - "space" => Some(0x31), - "`" => Some(0x32), - "delete" | "backspace" => Some(0x33), - "escape" | "esc" => Some(0x35), - "f1" => Some(0x7A), - "f2" => Some(0x78), - "f3" => Some(0x63), - "f4" => Some(0x76), - "f5" => Some(0x60), - "f6" => Some(0x61), - "f7" => Some(0x62), - "f8" => Some(0x64), - "f9" => Some(0x65), - "f10" => Some(0x6D), - "f11" => Some(0x67), - "f12" => Some(0x6F), - "left" => Some(0x7B), - "right" => Some(0x7C), - "down" => Some(0x7D), - "up" => Some(0x7E), - _ => None, - } - } -} - -#[cfg(target_os = "macos")] -struct ElementCollector<'a> { - role_filter: Option, - title_filter: Option, - identifier_filter: Option, - results: std::cell::RefCell<&'a mut Vec>, - depth: std::cell::Cell, -} - -#[cfg(target_os = "macos")] -impl<'a> TreeVisitor for ElementCollector<'a> { - fn enter_element(&self, element: &AXUIElement) -> TreeWalkerFlow { - self.depth.set(self.depth.get() + 1); - - if self.depth.get() > 20 { - return TreeWalkerFlow::SkipSubtree; - } - - // Get element properties - let role = element - .role() - .ok() - .map(|s| s.to_string()) - .unwrap_or_else(|| "Unknown".to_string()); - - let title = element.title().ok().map(|s| s.to_string()); - - let identifier = element.identifier().ok().map(|s| s.to_string()); - - // Check if this element matches the filters - let role_matches = self.role_filter.as_ref().map_or(true, |r| role.contains(r)); - let title_matches = self.title_filter.as_ref().map_or(true, |t| { - title - .as_ref() - .map_or(false, |title_str| title_str.contains(t)) - }); - let identifier_matches = self.identifier_filter.as_ref().map_or(true, |id| { - identifier - .as_ref() - .map_or(false, |id_str| id_str.contains(id)) - }); - - if role_matches && title_matches && identifier_matches { - // Get additional properties - let value = element - .value() - .ok() - .and_then(|v| v.downcast::().map(|s| s.to_string())); - - let label = element.description().ok().map(|s| s.to_string()); - - let enabled = element.enabled().ok().map(|b| b.into()).unwrap_or(false); - - let focused = element.focused().ok().map(|b| b.into()).unwrap_or(false); - - // Count children - let children_count = element - .children() - .ok() - .map(|arr| arr.len() as usize) - .unwrap_or(0); - - self.results.borrow_mut().push(AXElement { - role, - title, - value, - label, - identifier, - enabled, - focused, - position: None, - size: None, - children_count, - }); - } - - TreeWalkerFlow::Continue - } - - fn exit_element(&self, _element: &AXUIElement) { - self.depth.set(self.depth.get() - 1); - } -} diff --git a/crates/g3-computer-control/src/macax/mod.rs b/crates/g3-computer-control/src/macax/mod.rs deleted file mode 100644 index afe29a2..0000000 --- a/crates/g3-computer-control/src/macax/mod.rs +++ /dev/null @@ -1,65 +0,0 @@ -pub mod controller; - -pub use controller::MacAxController; - -use serde::{Deserialize, Serialize}; - -#[cfg(test)] -mod tests; - -/// Represents an accessibility element in the UI hierarchy -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AXElement { - pub role: String, - pub title: Option, - pub value: Option, - pub label: Option, - pub identifier: Option, - pub enabled: bool, - pub focused: bool, - pub position: Option<(f64, f64)>, - pub size: Option<(f64, f64)>, - pub children_count: usize, -} - -/// Represents a macOS application -#[derive(Debug, Clone)] -pub struct AXApplication { - pub name: String, - pub bundle_id: Option, - pub pid: i32, -} - -impl AXElement { - /// Convert to a human-readable string representation - pub fn to_string(&self) -> String { - let mut parts = vec![format!("Role: {}", self.role)]; - - if let Some(ref title) = self.title { - parts.push(format!("Title: {}", title)); - } - if let Some(ref value) = self.value { - parts.push(format!("Value: {}", value)); - } - if let Some(ref label) = self.label { - parts.push(format!("Label: {}", label)); - } - if let Some(ref id) = self.identifier { - parts.push(format!("ID: {}", id)); - } - - parts.push(format!("Enabled: {}", self.enabled)); - parts.push(format!("Focused: {}", self.focused)); - - if let Some((x, y)) = self.position { - parts.push(format!("Position: ({:.0}, {:.0})", x, y)); - } - if let Some((w, h)) = self.size { - parts.push(format!("Size: ({:.0}, {:.0})", w, h)); - } - - parts.push(format!("Children: {}", self.children_count)); - - parts.join(", ") - } -} diff --git a/crates/g3-computer-control/src/macax/tests.rs b/crates/g3-computer-control/src/macax/tests.rs deleted file mode 100644 index 01f44e3..0000000 --- a/crates/g3-computer-control/src/macax/tests.rs +++ /dev/null @@ -1,37 +0,0 @@ -#[cfg(test)] -mod tests { - use crate::{AXElement, MacAxController}; - - #[test] - fn test_ax_element_to_string() { - let element = AXElement { - role: "button".to_string(), - title: Some("Click Me".to_string()), - value: None, - label: Some("Submit Button".to_string()), - identifier: Some("submitBtn".to_string()), - enabled: true, - focused: false, - position: Some((100.0, 200.0)), - size: Some((80.0, 30.0)), - children_count: 0, - }; - - let string_repr = element.to_string(); - assert!(string_repr.contains("Role: button")); - assert!(string_repr.contains("Title: Click Me")); - assert!(string_repr.contains("Label: Submit Button")); - assert!(string_repr.contains("ID: submitBtn")); - assert!(string_repr.contains("Enabled: true")); - assert!(string_repr.contains("Position: (100, 200)")); - assert!(string_repr.contains("Size: (80, 30)")); - } - - #[test] - fn test_controller_creation() { - // Just test that we can create a controller - // Actual functionality requires macOS and permissions - let result = MacAxController::new(); - assert!(result.is_ok()); - } -} diff --git a/crates/g3-computer-control/src/platform/linux.rs b/crates/g3-computer-control/src/platform/linux.rs index 01db21b..4737f36 100644 --- a/crates/g3-computer-control/src/platform/linux.rs +++ b/crates/g3-computer-control/src/platform/linux.rs @@ -21,12 +21,4 @@ impl ComputerController for LinuxController { ) -> Result<()> { anyhow::bail!("Linux screenshot implementation not yet available") } - - fn move_mouse(&self, _x: i32, _y: i32) -> Result<()> { - anyhow::bail!("Linux mouse control not yet available") - } - - fn click_at(&self, _x: i32, _y: i32, _app_name: Option<&str>) -> Result<()> { - anyhow::bail!("Linux click control not yet available") - } } diff --git a/crates/g3-computer-control/src/platform/macos.rs b/crates/g3-computer-control/src/platform/macos.rs index 2b533aa..aa3fb5e 100644 --- a/crates/g3-computer-control/src/platform/macos.rs +++ b/crates/g3-computer-control/src/platform/macos.rs @@ -1,7 +1,7 @@ use crate::{ types::Rect, ComputerController, }; -use anyhow::{Context, Result}; +use anyhow::Result; use async_trait::async_trait; use core_foundation::array::CFArray; use core_foundation::base::{TCFType, ToVoid}; @@ -204,99 +204,8 @@ impl ComputerController for MacOSController { Ok(()) } - fn move_mouse(&self, x: i32, y: i32) -> Result<()> { - use core_graphics::event::{CGEvent, CGEventTapLocation, CGEventType, CGMouseButton}; - use core_graphics::event_source::{CGEventSource, CGEventSourceStateID}; - use core_graphics::geometry::CGPoint; - - let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) - .ok() - .context("Failed to create event source")?; - - let event = CGEvent::new_mouse_event( - source, - CGEventType::MouseMoved, - CGPoint::new(x as f64, y as f64), - CGMouseButton::Left, - ) - .ok() - .context("Failed to create mouse event")?; - - event.post(CGEventTapLocation::HID); - - Ok(()) - } - - fn click_at(&self, x: i32, y: i32, _app_name: Option<&str>) -> Result<()> { - use core_graphics::display::CGDisplay; - use core_graphics::event::{CGEvent, CGEventTapLocation, CGEventType, CGMouseButton}; - use core_graphics::event_source::{CGEventSource, CGEventSourceStateID}; - use core_graphics::geometry::CGPoint; - - // IMPORTANT: Coordinates passed here are in NSScreen/CGWindowListCopyWindowInfo space - // (Y=0 at BOTTOM, increases UPWARD) - // But CGEvent uses a different coordinate system (Y=0 at TOP, increases DOWNWARD) - // We need to convert: CGEvent.y = screenHeight - NSScreen.y - - let screen_height = CGDisplay::main().pixels_high() as i32; - let cgevent_x = x; - let cgevent_y = screen_height - y; - - tracing::debug!( - "click_at: NSScreen coords ({}, {}) -> CGEvent coords ({}, {}) [screen_height={}]", - x, - y, - cgevent_x, - cgevent_y, - screen_height - ); - - let (global_x, global_y) = (cgevent_x, cgevent_y); - - let point = CGPoint::new(global_x as f64, global_y as f64); - - let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) - .ok() - .context("Failed to create event source")?; - - // Move mouse to position first - let move_event = CGEvent::new_mouse_event( - source.clone(), - CGEventType::MouseMoved, - point, - CGMouseButton::Left, - ) - .ok() - .context("Failed to create mouse move event")?; - move_event.post(CGEventTapLocation::HID); - - std::thread::sleep(std::time::Duration::from_millis(100)); - - // Mouse down - let mouse_down = CGEvent::new_mouse_event( - source.clone(), - CGEventType::LeftMouseDown, - point, - CGMouseButton::Left, - ) - .ok() - .context("Failed to create mouse down event")?; - mouse_down.post(CGEventTapLocation::HID); - - std::thread::sleep(std::time::Duration::from_millis(50)); - - // Mouse up - let mouse_up = - CGEvent::new_mouse_event(source, CGEventType::LeftMouseUp, point, CGMouseButton::Left) - .ok() - .context("Failed to create mouse up event")?; - mouse_up.post(CGEventTapLocation::HID); - - Ok(()) - } } - #[path = "macos_window_matching_test.rs"] #[cfg(test)] mod tests; diff --git a/crates/g3-computer-control/src/platform/windows.rs b/crates/g3-computer-control/src/platform/windows.rs index a551fc0..e2f7acc 100644 --- a/crates/g3-computer-control/src/platform/windows.rs +++ b/crates/g3-computer-control/src/platform/windows.rs @@ -21,12 +21,4 @@ impl ComputerController for WindowsController { ) -> Result<()> { anyhow::bail!("Windows screenshot implementation not yet available") } - - fn move_mouse(&self, _x: i32, _y: i32) -> Result<()> { - anyhow::bail!("Windows mouse control not yet available") - } - - fn click_at(&self, _x: i32, _y: i32, _app_name: Option<&str>) -> Result<()> { - anyhow::bail!("Windows click control not yet available") - } }