168 lines
7.1 KiB
Rust
168 lines
7.1 KiB
Rust
use crate::{ComputerController, types::*};
|
|
use anyhow::Result;
|
|
use async_trait::async_trait;
|
|
use tesseract::Tesseract;
|
|
use uuid::Uuid;
|
|
|
|
pub struct WindowsController {
|
|
// Placeholder for Windows-specific state
|
|
}
|
|
|
|
impl WindowsController {
|
|
pub fn new() -> Result<Self> {
|
|
tracing::warn!("Windows computer control not fully implemented");
|
|
Ok(Self {})
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl ComputerController for WindowsController {
|
|
async fn move_mouse(&self, _x: i32, _y: i32) -> Result<()> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn click(&self, _button: MouseButton) -> Result<()> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn double_click(&self, _button: MouseButton) -> Result<()> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn type_text(&self, _text: &str) -> Result<()> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn press_key(&self, _key: &str) -> Result<()> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn list_windows(&self) -> Result<Vec<Window>> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn focus_window(&self, _window_id: &str) -> Result<()> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn get_window_bounds(&self, _window_id: &str) -> Result<Rect> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn find_element(&self, _selector: &ElementSelector) -> Result<Option<UIElement>> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn get_element_text(&self, _element_id: &str) -> Result<String> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn get_element_bounds(&self, _element_id: &str) -> Result<Rect> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn take_screenshot(&self, _path: &str, _region: Option<Rect>, _window_id: Option<&str>) -> Result<()> {
|
|
// Enforce that window_id must be provided
|
|
if _window_id.is_none() {
|
|
anyhow::bail!("window_id is required. You must specify which window to capture (e.g., 'Chrome', 'Terminal', 'Notepad'). Use list_windows to see available windows.");
|
|
}
|
|
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn extract_text_from_screen(&self, _region: Rect, _window_id: &str) -> Result<String> {
|
|
anyhow::bail!("Windows implementation not yet available")
|
|
}
|
|
|
|
async fn extract_text_from_image(&self, _path: &str) -> Result<OCRResult> {
|
|
// Check if tesseract is available on the system
|
|
let tesseract_check = std::process::Command::new("where")
|
|
.arg("tesseract")
|
|
.output();
|
|
|
|
if tesseract_check.is_err() || !tesseract_check.as_ref().unwrap().status.success() {
|
|
anyhow::bail!("Tesseract OCR is not installed on your system.\n\n\
|
|
To install tesseract on Windows:\n \
|
|
1. Download the installer from: https://github.com/UB-Mannheim/tesseract/wiki\n \
|
|
2. Run the installer and follow the instructions\n \
|
|
3. Add tesseract to your PATH environment variable\n \
|
|
4. Restart your terminal/command prompt\n\n\
|
|
After installation, restart your terminal and try again.");
|
|
}
|
|
|
|
// Initialize Tesseract
|
|
let tess = Tesseract::new(None, Some("eng"))
|
|
.map_err(|e| {
|
|
anyhow::anyhow!("Failed to initialize Tesseract: {}\n\n\
|
|
This usually means:\n1. Tesseract is not properly installed\n\
|
|
2. Language data files are missing\n\nTo fix:\n \
|
|
1. Reinstall tesseract from https://github.com/UB-Mannheim/tesseract/wiki\n \
|
|
2. Make sure to select 'Additional language data' during installation\n \
|
|
3. Ensure tesseract is in your PATH", e)
|
|
})?;
|
|
|
|
let text = tess.set_image(_path)
|
|
.map_err(|e| anyhow::anyhow!("Failed to load image '{}': {}", _path, e))?
|
|
.get_text()
|
|
.map_err(|e| anyhow::anyhow!("Failed to extract text from image: {}", e))?;
|
|
|
|
// Get confidence (simplified - would need more complex API calls for per-word confidence)
|
|
let confidence = 0.85; // Placeholder
|
|
|
|
Ok(OCRResult {
|
|
text,
|
|
confidence,
|
|
bounds: Rect { x: 0, y: 0, width: 0, height: 0 }, // Would need image dimensions
|
|
})
|
|
}
|
|
|
|
async fn find_text_on_screen(&self, _text: &str) -> Result<Option<Point>> {
|
|
// Check if tesseract is available on the system
|
|
let tesseract_check = std::process::Command::new("where")
|
|
.arg("tesseract")
|
|
.output();
|
|
|
|
if tesseract_check.is_err() || !tesseract_check.as_ref().unwrap().status.success() {
|
|
anyhow::bail!("Tesseract OCR is not installed on your system.\n\n\
|
|
To install tesseract on Windows:\n \
|
|
1. Download the installer from: https://github.com/UB-Mannheim/tesseract/wiki\n \
|
|
2. Run the installer and follow the instructions\n \
|
|
3. Add tesseract to your PATH environment variable\n \
|
|
4. Restart your terminal/command prompt\n\n\
|
|
After installation, restart your terminal and try again.");
|
|
}
|
|
|
|
// Take full screen screenshot
|
|
let temp_path = format!("C:\\\\Temp\\\\g3_ocr_search_{}.png", uuid::Uuid::new_v4());
|
|
self.take_screenshot(&temp_path, None, None).await?;
|
|
|
|
// Use Tesseract to find text with bounding boxes
|
|
let tess = Tesseract::new(None, Some("eng"))
|
|
.map_err(|e| {
|
|
anyhow::anyhow!("Failed to initialize Tesseract: {}\n\n\
|
|
This usually means:\n1. Tesseract is not properly installed\n\
|
|
2. Language data files are missing\n\nTo fix:\n \
|
|
1. Reinstall tesseract from https://github.com/UB-Mannheim/tesseract/wiki\n \
|
|
2. Make sure to select 'Additional language data' during installation\n \
|
|
3. Ensure tesseract is in your PATH", e)
|
|
})?;
|
|
|
|
let full_text = tess.set_image(temp_path.as_str())
|
|
.map_err(|e| anyhow::anyhow!("Failed to load screenshot: {}", e))?
|
|
.get_text()
|
|
.map_err(|e| anyhow::anyhow!("Failed to extract text from screen: {}", e))?;
|
|
|
|
// Clean up temp file
|
|
let _ = std::fs::remove_file(&temp_path);
|
|
|
|
// Simple text search - full implementation would use get_component_images
|
|
// to get bounding boxes for each word
|
|
if full_text.contains(_text) {
|
|
tracing::warn!("Text found but precise coordinates not available in simplified implementation");
|
|
Ok(Some(Point { x: 0, y: 0 }))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
}
|