replace tesseract with apple vision

crates/g3-computer-control/src/ocr/mod.rs

@@ -0,0 +1,26 @@
+use crate::types::TextLocation;
+use anyhow::Result;
+use async_trait::async_trait;
+
+/// OCR engine trait for text recognition with bounding boxes
+#[async_trait]
+pub trait OCREngine: Send + Sync {
+    /// Extract text with locations from an image file
+    async fn extract_text_with_locations(&self, path: &str) -> Result<Vec<TextLocation>>;
+    
+    /// Get the name of the OCR engine
+    fn name(&self) -> &str;
+}
+
+// Platform-specific modules
+#[cfg(target_os = "macos")]
+pub mod vision;
+
+pub mod tesseract;
+
+// Re-export the default OCR engine for the platform
+#[cfg(target_os = "macos")]
+pub use vision::AppleVisionOCR as DefaultOCR;
+
+#[cfg(not(target_os = "macos"))]
+pub use tesseract::TesseractOCR as DefaultOCR;

crates/g3-computer-control/src/ocr/tesseract.rs

@@ -0,0 +1,84 @@
+use super::OCREngine;
+use crate::types::TextLocation;
+use anyhow::Result;
+use async_trait::async_trait;
+
+/// Tesseract OCR engine (fallback/cross-platform)
+pub struct TesseractOCR;
+
+impl TesseractOCR {
+    pub fn new() -> Result<Self> {
+        // Check if tesseract is available
+        let tesseract_check = std::process::Command::new("which")
+            .arg("tesseract")
+            .output();
+        
+        if tesseract_check.is_err() || !tesseract_check.as_ref().unwrap().status.success() {
+            anyhow::bail!("Tesseract OCR is not installed on your system.\n\n\
+                To install tesseract:\n  macOS:   brew install tesseract\n  \
+                Linux:   sudo apt-get install tesseract-ocr (Ubuntu/Debian)\n           \
+                sudo yum install tesseract (RHEL/CentOS)\n  \
+                Windows: Download from https://github.com/UB-Mannheim/tesseract/wiki\n\n\
+                After installation, restart your terminal and try again.");
+        }
+        
+        Ok(Self)
+    }
+}
+
+#[async_trait]
+impl OCREngine for TesseractOCR {
+    async fn extract_text_with_locations(&self, path: &str) -> Result<Vec<TextLocation>> {
+        // Use tesseract CLI with TSV output to get bounding boxes
+        let output = std::process::Command::new("tesseract")
+            .arg(path)
+            .arg("stdout")
+            .arg("tsv")
+            .output()
+            .map_err(|e| anyhow::anyhow!("Failed to run tesseract: {}", e))?;
+        
+        if !output.status.success() {
+            anyhow::bail!("Tesseract failed: {}", String::from_utf8_lossy(&output.stderr));
+        }
+        
+        let tsv_text = String::from_utf8_lossy(&output.stdout);
+        let mut locations = Vec::new();
+        
+        // Parse TSV output (skip header line)
+        for (i, line) in tsv_text.lines().enumerate() {
+            if i == 0 { continue; } // Skip header
+            
+            let parts: Vec<&str> = line.split('\t').collect();
+            if parts.len() >= 12 {
+                // TSV format: level, page_num, block_num, par_num, line_num, word_num,
+                //             left, top, width, height, conf, text
+                if let (Ok(x), Ok(y), Ok(w), Ok(h), Ok(conf), text) = (
+                    parts[6].parse::<i32>(),
+                    parts[7].parse::<i32>(),
+                    parts[8].parse::<i32>(),
+                    parts[9].parse::<i32>(),
+                    parts[10].parse::<f32>(),
+                    parts[11],
+                ) {
+                    let trimmed = text.trim();
+                    if !trimmed.is_empty() && conf > 0.0 {
+                        locations.push(TextLocation {
+                            text: trimmed.to_string(),
+                            x,
+                            y,
+                            width: w,
+                            height: h,
+                            confidence: conf / 100.0, // Convert from 0-100 to 0-1
+                        });
+                    }
+                }
+            }
+        }
+        
+        Ok(locations)
+    }
+    
+    fn name(&self) -> &str {
+        "Tesseract OCR"
+    }
+}

crates/g3-computer-control/src/ocr/vision.rs

@@ -0,0 +1,103 @@
+use super::OCREngine;
+use crate::types::TextLocation;
+use anyhow::{Result, Context};
+use async_trait::async_trait;
+use std::ffi::{CStr, CString};
+use std::os::raw::{c_char, c_float, c_uint};
+
+// FFI bindings to Swift VisionBridge
+#[repr(C)]
+struct VisionTextBox {
+    text: *const c_char,
+    text_len: c_uint,
+    x: i32,
+    y: i32,
+    width: i32,
+    height: i32,
+    confidence: c_float,
+}
+
+extern "C" {
+    fn vision_recognize_text(
+        image_path: *const c_char,
+        image_path_len: c_uint,
+        out_boxes: *mut *mut std::ffi::c_void,
+        out_count: *mut c_uint,
+    ) -> bool;
+    
+    fn vision_free_boxes(boxes: *mut std::ffi::c_void, count: c_uint);
+}
+
+/// Apple Vision Framework OCR engine
+pub struct AppleVisionOCR;
+
+impl AppleVisionOCR {
+    pub fn new() -> Result<Self> {
+        Ok(Self)
+    }
+}
+
+#[async_trait]
+impl OCREngine for AppleVisionOCR {
+    async fn extract_text_with_locations(&self, path: &str) -> Result<Vec<TextLocation>> {
+        // Convert path to C string
+        let c_path = CString::new(path)
+            .context("Failed to convert path to C string")?;
+        
+        let mut boxes_ptr: *mut std::ffi::c_void = std::ptr::null_mut();
+        let mut count: c_uint = 0;
+        
+        // Call Swift Vision API
+        let success = unsafe {
+            vision_recognize_text(
+                c_path.as_ptr(),
+                path.len() as c_uint,
+                &mut boxes_ptr,
+                &mut count,
+            )
+        };
+        
+        if !success || boxes_ptr.is_null() {
+            anyhow::bail!("Apple Vision OCR failed");
+        }
+        
+        // Convert C array to Rust Vec
+        let mut locations = Vec::new();
+        
+        unsafe {
+            let typed_boxes = boxes_ptr as *const VisionTextBox;
+            let boxes_slice = std::slice::from_raw_parts(typed_boxes, count as usize);
+            
+            for box_data in boxes_slice {
+                // Convert C string to Rust String
+                let text = if !box_data.text.is_null() {
+                    CStr::from_ptr(box_data.text)
+                        .to_string_lossy()
+                        .into_owned()
+                } else {
+                    String::new()
+                };
+                
+                if !text.is_empty() {
+                    locations.push(TextLocation {
+                        text,
+                        x: box_data.x,
+                        y: box_data.y,
+                        width: box_data.width,
+                        height: box_data.height,
+                        confidence: box_data.confidence,
+                    });
+                }
+            }
+            
+            // Free the C array
+            vision_free_boxes(boxes_ptr, count);
+        }
+        
+        Ok(locations)
+    }
+    
+    fn name(&self) -> &str {
+        "Apple Vision Framework"
+    }
+}