alex/g3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into micn/libvision-fix

Browse Source 
* main:
  control commands for machine mode
  Fix duplicate dump at end
  minor
  --machine mode flag for verbose CLI output
  fixed x,y detection in vision click
  screenshotting bug fix
This commit is contained in:
Michael Neale
2025-10-28 13:56:49 +11:00
parent 8a7d6ad4ba 7c2c433746
commit f7d1c8e42a
11 changed files with 964 additions and 1023 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
Cargo.lock generated
View File

@@ -318,9 +318,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.41"
version = "1.2.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac9fe6cdbb24b6ade63616c0a0688e45bb56732262c158df3c0c4bea4ca47cb7"
checksum = "739eb0f94557554b3ca9a86d2d37bebd49c5e6d0c1d2bda35ba5bdac830befc2"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -900,9 +900,9 @@ dependencies = [
[[package]]
name = "deranged"
version = "0.5.4"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071"
checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587"
dependencies = [
"powerfmt",
]
@@ -990,7 +990,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users 0.5.2",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -1015,9 +1015,9 @@ dependencies = [
[[package]]
name = "document-features"
version = "0.2.11"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d"
checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
dependencies = [
"litrs",
]
@@ -1062,7 +1062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -1144,9 +1144,9 @@ checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"
[[package]]
name = "flate2"
version = "1.1.4"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9"
checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
dependencies = [
"crc32fast",
"miniz_oxide",
@@ -1571,11 +1571,11 @@ checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
[[package]]
name = "home"
version = "0.5.11"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"
checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -1922,9 +1922,12 @@ dependencies = [
[[package]]
name = "indoc"
version = "2.0.6"
version = "2.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd"
checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
dependencies = [
"rustversion",
]
[[package]]
name = "instability"
@@ -1947,9 +1950,9 @@ checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]]
name = "itertools"
@@ -2133,9 +2136,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
[[package]]
name = "litrs"
version = "0.4.2"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed"
checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
[[package]]
name = "llama_cpp"
@@ -2251,14 +2254,14 @@ dependencies = [
[[package]]
name = "mio"
version = "1.0.4"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873"
dependencies = [
"libc",
"log",
"wasi",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -2330,7 +2333,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -2406,9 +2409,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "once_cell_polyfill"
version = "1.70.1"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "openssl"
@@ -2627,9 +2630,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.101"
version = "1.0.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8"
dependencies = [
"unicode-ident",
]
@@ -2901,7 +2904,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.11.0",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -3122,9 +3125,9 @@ dependencies = [
[[package]]
name = "signal-hook-mio"
version = "0.2.4"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc"
dependencies = [
"libc",
"mio",
@@ -3226,9 +3229,9 @@ dependencies = [
[[package]]
name = "syn"
version = "2.0.107"
version = "2.0.108"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a26dbd934e5451d21ef060c018dae56fc073894c5a7896f882928a76e6d081b"
checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
dependencies = [
"proc-macro2",
"quote",
@@ -3289,7 +3292,7 @@ dependencies = [
"getrandom 0.3.4",
"once_cell",
"rustix 1.1.2",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -3631,9 +3634,9 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
[[package]]
name = "unicode-ident"
version = "1.0.19"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d"
checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06"
[[package]]
name = "unicode-segmentation"
@@ -3932,7 +3935,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.48.0",
]
[[package]]

686
crates/g3-cli/src/lib.rs
View File

@@ -167,14 +167,12 @@ use tokio_util::sync::CancellationToken;
use tracing::{error, info};
use g3_core::error_handling::{classify_error, ErrorType, RecoverableError};
mod retro_tui;
mod theme;
pub mod tui;
mod ui_writer_impl;
use retro_tui::RetroTui;
use theme::ColorTheme;
use tui::SimpleOutput;
use ui_writer_impl::{ConsoleUiWriter, RetroTuiWriter};
mod simple_output;
use simple_output::SimpleOutput;
mod machine_ui_writer;
use machine_ui_writer::MachineUiWriter;
use ui_writer_impl::ConsoleUiWriter;
#[derive(Parser)]
#[command(name = "g3")]
@@ -220,13 +218,9 @@ pub struct Cli {
#[arg(long)]
pub interactive_requirements: bool,
/// Use retro terminal UI (inspired by 80s sci-fi)
/// Enable machine-friendly output mode with JSON markers and stats
#[arg(long)]
pub retro: bool,
/// Color theme for retro mode (default, dracula, or path to theme file)
#[arg(long, value_name = "THEME")]
pub theme: Option<String>,
pub machine: bool,
/// Override the configured provider (anthropic, databricks, embedded, openai)
#[arg(long, value_name = "PROVIDER")]
@@ -253,7 +247,7 @@ pub async fn run() -> Result<()> {
let cli = Cli::parse();
// Only initialize logging if not in retro mode
if !cli.retro {
if !cli.machine {
// Initialize logging with filtering
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
@@ -291,16 +285,16 @@ pub async fn run() -> Result<()> {
tracing_subscriber::registry().with(filter).init();
}
if !cli.retro {
if !cli.machine {
info!("Starting G3 AI Coding Agent");
}
// Set up workspace directory
let workspace_dir = if let Some(ws) = cli.workspace {
ws
let workspace_dir = if let Some(ws) = &cli.workspace {
ws.clone()
} else if cli.autonomous {
// For autonomous mode, use G3_WORKSPACE env var or default
setup_workspace_directory()?
setup_workspace_directory(cli.machine)?
} else {
// Default to current directory for interactive/single-shot mode
std::env::current_dir()?
@@ -421,9 +415,9 @@ Output ONLY the markdown content, no explanations or meta-commentary."#,
}
}
if let Some(requirements_text) = cli.requirements {
if let Some(requirements_text) = &cli.requirements {
// Use requirements text override
Project::new_autonomous_with_requirements(workspace_dir.clone(), requirements_text)?
Project::new_autonomous_with_requirements(workspace_dir.clone(), requirements_text.clone())?
} else {
// Use traditional requirements.md file
Project::new_autonomous(workspace_dir.clone())?
@@ -436,7 +430,7 @@ Output ONLY the markdown content, no explanations or meta-commentary."#,
project.ensure_workspace_exists()?;
project.enter_workspace()?;
if !cli.retro {
if !cli.machine {
info!("Using workspace: {}", project.workspace().display());
}
@@ -450,7 +444,7 @@ Output ONLY the markdown content, no explanations or meta-commentary."#,
// Apply macax flag override
if cli.macax {
config.macax.enabled = true;
if !cli.retro {
if !cli.machine {
info!("macOS Accessibility API tools enabled");
}
}
@@ -473,7 +467,7 @@ Output ONLY the markdown content, no explanations or meta-commentary."#,
}
// Initialize agent
let ui_writer = ConsoleUiWriter::new();
// ui_writer will be created conditionally based on machine mode
// Combine AGENTS.md and README content if both exist
let combined_content = match (agents_content.clone(), readme_content.clone()) {
@@ -485,28 +479,117 @@ Output ONLY the markdown content, no explanations or meta-commentary."#,
(None, None) => None,
};
let mut agent = if cli.autonomous {
Agent::new_autonomous_with_readme_and_quiet(
config.clone(),
ui_writer,
combined_content.clone(),
cli.quiet,
)
.await?
// Execute task, autonomous mode, or start interactive mode based on machine mode
if cli.machine {
// Machine mode - use MachineUiWriter
let ui_writer = MachineUiWriter::new();
let agent = if cli.autonomous {
Agent::new_autonomous_with_readme_and_quiet(
config.clone(),
ui_writer,
combined_content.clone(),
cli.quiet,
)
.await?
} else {
Agent::new_with_readme_and_quiet(
config.clone(),
ui_writer,
combined_content.clone(),
cli.quiet,
)
.await?
};
run_with_machine_mode(agent, cli, project).await?;
} else {
Agent::new_with_readme_and_quiet(
config.clone(),
ui_writer,
combined_content.clone(),
cli.quiet,
)
.await?
// Normal mode - use ConsoleUiWriter
let ui_writer = ConsoleUiWriter::new();
let agent = if cli.autonomous {
Agent::new_autonomous_with_readme_and_quiet(
config.clone(),
ui_writer,
combined_content.clone(),
cli.quiet,
)
.await?
} else {
Agent::new_with_readme_and_quiet(
config.clone(),
ui_writer,
combined_content.clone(),
cli.quiet,
)
.await?
};
run_with_console_mode(agent, cli, project, combined_content).await?;
}
Ok(())
}
// Simplified machine mode version of autonomous mode
async fn run_autonomous_machine(
mut agent: Agent<MachineUiWriter>,
project: Project,
show_prompt: bool,
show_code: bool,
max_turns: usize,
_quiet: bool,
) -> Result<()> {
println!("AUTONOMOUS_MODE_STARTED");
println!("WORKSPACE: {}", project.workspace().display());
println!("MAX_TURNS: {}", max_turns);
// Check if requirements exist
if !project.has_requirements() {
println!("ERROR: requirements.md not found in workspace directory");
return Ok(());
}
// Read requirements
let requirements = match project.read_requirements()? {
Some(content) => content,
None => {
println!("ERROR: Could not read requirements");
return Ok(());
}
};
println!("REQUIREMENTS_LOADED");
// For now, just execute a simple autonomous loop
// This is a simplified version - full implementation would need coach-player loop
let task = format!(
"You are G3 in implementation mode. Read and implement the following requirements:\n\n{}\n\nImplement this step by step, creating all necessary files and code.",
requirements
);
println!("TASK_START");
let result = agent.execute_task_with_timing(&task, None, false, show_prompt, show_code, true).await?;
println!("AGENT_RESPONSE:");
println!("{}", result.response);
println!("END_AGENT_RESPONSE");
println!("TASK_END");
println!("AUTONOMOUS_MODE_ENDED");
Ok(())
}
async fn run_with_console_mode(
mut agent: Agent<ConsoleUiWriter>,
cli: Cli,
project: Project,
combined_content: Option<String>,
) -> Result<()> {
// Execute task, autonomous mode, or start interactive mode
if cli.autonomous {
// Autonomous mode with coach-player feedback loop
if !cli.retro {
if !cli.machine {
info!("Starting autonomous mode");
}
run_autonomous(
@@ -520,7 +603,7 @@ Output ONLY the markdown content, no explanations or meta-commentary."#,
.await?;
} else if let Some(task) = cli.task {
// Single-shot mode
if !cli.retro {
if !cli.machine {
info!("Executing task: {}", task);
}
let output = SimpleOutput::new();
@@ -530,26 +613,43 @@ Output ONLY the markdown content, no explanations or meta-commentary."#,
output.print_smart(&result.response);
} else {
// Interactive mode (default)
if !cli.retro {
if !cli.machine {
info!("Starting interactive mode");
}
println!("📁 Workspace: {}", project.workspace().display());
run_interactive(agent, cli.show_prompt, cli.show_code, combined_content).await?;
}
if cli.retro {
// Use retro terminal UI
run_interactive_retro(
config, // Already has overrides applied
cli.show_prompt,
cli.show_code,
cli.theme,
combined_content,
)
Ok(())
}
async fn run_with_machine_mode(
mut agent: Agent<MachineUiWriter>,
cli: Cli,
project: Project,
) -> Result<()> {
if cli.autonomous {
// Autonomous mode with coach-player feedback loop
run_autonomous_machine(
agent,
project,
cli.show_prompt,
cli.show_code,
cli.max_turns,
cli.quiet,
)
.await?;
} else if let Some(task) = cli.task {
// Single-shot mode
let result = agent
.execute_task_with_timing(&task, None, false, cli.show_prompt, cli.show_code, true)
.await?;
} else {
// Use standard terminal UI
let output = SimpleOutput::new();
output.print(&format!("📁 Workspace: {}", project.workspace().display()));
run_interactive(agent, cli.show_prompt, cli.show_code, combined_content).await?;
}
println!("AGENT_RESPONSE:");
println!("{}", result.response);
println!("END_AGENT_RESPONSE");
} else {
// Interactive mode
run_interactive_machine(agent, cli.show_prompt, cli.show_code).await?;
}
Ok(())
@@ -691,274 +791,6 @@ fn extract_readme_heading(readme_content: &str) -> Option<String> {
None
}
async fn run_interactive_retro(
config: Config,
show_prompt: bool,
show_code: bool,
theme_name: Option<String>,
combined_content: Option<String>,
) -> Result<()> {
use crossterm::event::{self, Event, KeyCode, KeyModifiers};
use std::time::Duration;
// Set environment variable to suppress println in other crates
std::env::set_var("G3_RETRO_MODE", "1");
// Load the color theme
let theme = match ColorTheme::load(theme_name.as_deref()) {
Ok(t) => t,
Err(e) => {
eprintln!("Failed to load theme: {}. Using default.", e);
ColorTheme::default()
}
};
// Initialize the retro terminal UI
let tui = RetroTui::start(theme).await?;
// Create agent with RetroTuiWriter
let ui_writer = RetroTuiWriter::new(tui.clone());
let mut agent = Agent::new_with_readme_and_quiet(config, ui_writer, combined_content.clone(), false).await?;
// Display initial system messages
tui.output("SYSTEM: AGENT ONLINE\n\n");
// Display message if AGENTS.md or README was loaded
if let Some(ref content) = combined_content {
// Check what was loaded
let has_agents = content.contains("Agent Configuration");
let has_readme = content.contains("Project README");
if has_agents {
tui.output("SYSTEM: AGENT CONFIGURATION LOADED\n\n");
}
if has_readme {
// Extract the first heading or title from the README
let readme_snippet = extract_readme_heading(content)
.unwrap_or_else(|| "PROJECT DOCUMENTATION LOADED".to_string());
tui.output(&format!(
"SYSTEM: PROJECT README LOADED - {}\n\n",
readme_snippet
));
}
}
tui.output("SYSTEM: READY FOR INPUT\n\n");
tui.output("\n\n");
// Display provider and model information
match agent.get_provider_info() {
Ok((provider, model)) => {
tui.update_provider_info(&provider, &model);
}
Err(e) => {
tui.update_provider_info("ERROR", &e.to_string());
}
}
// Track multiline input
let mut multiline_buffer = String::new();
let mut in_multiline = false;
// Main event loop
loop {
// Update context window display
let context = agent.get_context_window();
tui.update_context(
context.used_tokens,
context.total_tokens,
context.percentage_used(),
);
// Poll for keyboard events
if event::poll(Duration::from_millis(50))? {
if let Event::Key(key) = event::read()? {
match key.code {
KeyCode::Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.exit();
break;
}
KeyCode::Char('d') if key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.exit();
break;
}
// Emacs/bash-like shortcuts
KeyCode::Char('a') if key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.cursor_home();
}
KeyCode::Char('e') if key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.cursor_end();
}
KeyCode::Char('w') if key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.delete_word();
}
KeyCode::Char('k') if key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.delete_to_end();
}
KeyCode::Char('u') if key.modifiers.contains(KeyModifiers::CONTROL) => {
// Delete from beginning to cursor (similar to Ctrl-K but opposite direction)
let (input_buffer, cursor_pos) = tui.get_input_state();
if cursor_pos > 0 {
let after = input_buffer.chars().skip(cursor_pos).collect::<String>();
tui.update_input(&after);
tui.cursor_home();
}
}
KeyCode::Left => {
tui.cursor_left();
}
KeyCode::Right => {
tui.cursor_right();
}
KeyCode::Home if !key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.cursor_home();
}
KeyCode::End if !key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.cursor_end();
}
KeyCode::Delete => {
tui.delete_char();
}
KeyCode::Enter => {
let (input_buffer, _) = tui.get_input_state();
if !input_buffer.is_empty() {
// Clear the input for next command
tui.update_input("");
let trimmed = input_buffer.trim_end();
// Check if line ends with backslash for continuation
if let Some(without_backslash) = trimmed.strip_suffix('\\') {
// Remove the backslash and add to buffer
multiline_buffer.push_str(without_backslash);
multiline_buffer.push('\n');
in_multiline = true;
tui.status("MULTILINE INPUT");
continue;
}
// If we're in multiline mode and no backslash, this is the final line
let final_input = if in_multiline {
multiline_buffer.push_str(&input_buffer);
in_multiline = false;
let result = multiline_buffer.clone();
multiline_buffer.clear();
tui.status("READY");
result
} else {
input_buffer.clone()
};
let input = final_input.trim().to_string();
if input.is_empty() {
continue;
}
if input == "exit" || input == "quit" {
tui.exit();
break;
}
// Execute the task
tui.output(&format!("> {}", input));
tui.status("PROCESSING");
const MAX_TIMEOUT_RETRIES: u32 = 3;
let mut attempt = 0;
loop {
attempt += 1;
match agent
.execute_task_with_timing(
&input,
None,
false,
show_prompt,
show_code,
true,
)
.await
{
Ok(result) => {
if attempt > 1 {
tui.output(&format!(
"SYSTEM: REQUEST SUCCEEDED AFTER {} ATTEMPTS",
attempt
));
}
tui.output(&result.response);
tui.status("READY");
break;
}
Err(e) => {
// Check if this is a timeout error that we should retry
let error_type = classify_error(&e);
if matches!(
error_type,
ErrorType::Recoverable(RecoverableError::Timeout)
) && attempt < MAX_TIMEOUT_RETRIES
{
// Calculate retry delay with exponential backoff
let delay_ms = 1000 * (2_u64.pow(attempt - 1));
let delay = std::time::Duration::from_millis(delay_ms);
tui.output(&format!("SYSTEM: TIMEOUT ERROR (ATTEMPT {}/{}). RETRYING IN {:?}...",
attempt, MAX_TIMEOUT_RETRIES, delay));
tui.status("RETRYING");
// Wait before retrying
tokio::time::sleep(delay).await;
continue;
}
// For non-timeout errors or after max retries
tui.error(&format!("Task execution failed: {}", e));
tui.status("ERROR");
break;
}
}
}
}
}
KeyCode::Char(c) => {
tui.insert_char(c);
}
KeyCode::Backspace => {
tui.backspace();
}
KeyCode::Up => {
tui.scroll_up();
}
KeyCode::Down => {
tui.scroll_down();
}
KeyCode::PageUp => {
tui.scroll_page_up();
}
KeyCode::PageDown => {
tui.scroll_page_down();
}
KeyCode::Home if key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.scroll_home(); // Ctrl+Home for scrolling to top
}
KeyCode::End if key.modifiers.contains(KeyModifiers::CONTROL) => {
tui.scroll_end(); // Ctrl+End for scrolling to bottom
}
_ => {}
}
}
}
// Small delay to prevent CPU spinning
tokio::time::sleep(Duration::from_millis(10)).await;
}
tui.output("SYSTEM: SHUTDOWN INITIATED");
Ok(())
}
async fn run_interactive<W: UiWriter>(
mut agent: Agent<W>,
show_prompt: bool,
@@ -1109,7 +941,7 @@ async fn run_interactive<W: UiWriter>(
}
"/thinnify" => {
let summary = agent.force_thin();
output.print_context_thinning(&summary);
println!("{}", summary);
continue;
}
"/readme" => {
@@ -1247,6 +1079,199 @@ async fn execute_task<W: UiWriter>(
}
}
async fn run_interactive_machine(
mut agent: Agent<MachineUiWriter>,
show_prompt: bool,
show_code: bool,
) -> Result<()> {
println!("INTERACTIVE_MODE_STARTED");
// Display provider and model information
match agent.get_provider_info() {
Ok((provider, model)) => {
println!("PROVIDER: {}", provider);
println!("MODEL: {}", model);
}
Err(e) => {
println!("ERROR: Failed to get provider info: {}", e);
}
}
// Initialize rustyline editor with history
let mut rl = DefaultEditor::new()?;
// Try to load history from a file in the user's home directory
let history_file = dirs::home_dir().map(|mut path| {
path.push(".g3_history");
path
});
if let Some(ref history_path) = history_file {
let _ = rl.load_history(history_path);
}
loop {
let readline = rl.readline("");
match readline {
Ok(line) => {
let input = line.trim().to_string();
if input.is_empty() {
continue;
}
if input == "exit" || input == "quit" {
break;
}
// Add to history
rl.add_history_entry(&input)?;
// Check for control commands
if input.starts_with('/') {
match input.as_str() {
"/compact" => {
println!("COMMAND: compact");
match agent.force_summarize().await {
Ok(true) => println!("RESULT: Summarization completed"),
Ok(false) => println!("RESULT: Summarization failed"),
Err(e) => println!("ERROR: {}", e),
}
continue;
}
"/thinnify" => {
println!("COMMAND: thinnify");
let summary = agent.force_thin();
println!("{}", summary);
continue;
}
"/readme" => {
println!("COMMAND: readme");
match agent.reload_readme() {
Ok(true) => println!("RESULT: README content reloaded successfully"),
Ok(false) => println!("RESULT: No README was loaded at startup, cannot reload"),
Err(e) => println!("ERROR: {}", e),
}
continue;
}
"/stats" => {
println!("COMMAND: stats");
let stats = agent.get_stats();
// Emit stats as structured data (name: value pairs)
println!("{}", stats);
continue;
}
"/help" => {
println!("COMMAND: help");
println!("AVAILABLE_COMMANDS: /compact /thinnify /readme /stats /help");
continue;
}
_ => {
println!("ERROR: Unknown command: {}", input);
continue;
}
}
}
// Execute task
println!("TASK_START");
execute_task_machine(&mut agent, &input, show_prompt, show_code).await;
println!("TASK_END");
}
Err(ReadlineError::Interrupted) => continue,
Err(ReadlineError::Eof) => break,
Err(err) => {
println!("ERROR: {:?}", err);
break;
}
}
}
// Save history before exiting
if let Some(ref history_path) = history_file {
let _ = rl.save_history(history_path);
}
println!("INTERACTIVE_MODE_ENDED");
Ok(())
}
async fn execute_task_machine(
agent: &mut Agent<MachineUiWriter>,
input: &str,
show_prompt: bool,
show_code: bool,
) {
const MAX_TIMEOUT_RETRIES: u32 = 3;
let mut attempt = 0;
// Create cancellation token for this request
let cancellation_token = CancellationToken::new();
let cancel_token_clone = cancellation_token.clone();
loop {
attempt += 1;
// Execute task with cancellation support
let execution_result = tokio::select! {
result = agent.execute_task_with_timing_cancellable(
input, None, false, show_prompt, show_code, true, cancellation_token.clone()
) => {
result
}
_ = tokio::signal::ctrl_c() => {
cancel_token_clone.cancel();
println!("CANCELLED");
return;
}
};
match execution_result {
Ok(result) => {
if attempt > 1 {
println!("RETRY_SUCCESS: attempt {}", attempt);
}
println!("AGENT_RESPONSE:");
println!("{}", result.response);
println!("END_AGENT_RESPONSE");
return;
}
Err(e) => {
if e.to_string().contains("cancelled") {
println!("CANCELLED");
return;
}
// Check if this is a timeout error that we should retry
let error_type = classify_error(&e);
if matches!(
error_type,
ErrorType::Recoverable(RecoverableError::Timeout)
) && attempt < MAX_TIMEOUT_RETRIES
{
// Calculate retry delay with exponential backoff
let delay_ms = 1000 * (2_u64.pow(attempt - 1));
let delay = std::time::Duration::from_millis(delay_ms);
println!("TIMEOUT: attempt {} of {}, retrying in {:?}", attempt, MAX_TIMEOUT_RETRIES, delay);
// Wait before retrying
tokio::time::sleep(delay).await;
continue;
}
// For non-timeout errors or after max retries
println!("ERROR: {}", e);
if attempt > 1 {
println!("FAILED_AFTER_RETRIES: {}", attempt);
}
return;
}
}
}
}
fn handle_execution_error(e: &anyhow::Error, input: &str, output: &SimpleOutput, attempt: u32) {
// Enhanced error logging with detailed information
error!("=== TASK EXECUTION ERROR ===");
@@ -1280,16 +1305,13 @@ fn handle_execution_error(e: &anyhow::Error, input: &str, output: &SimpleOutput,
fn display_context_progress<W: UiWriter>(agent: &Agent<W>, output: &SimpleOutput) {
let context = agent.get_context_window();
output.print_context(
context.used_tokens,
context.total_tokens,
context.percentage_used(),
);
output.print(&format!("Context: {}/{} tokens ({:.1}%)",
context.used_tokens, context.total_tokens, context.percentage_used()));
}
/// Set up the workspace directory for autonomous mode
/// Uses G3_WORKSPACE environment variable or defaults to ~/tmp/workspace
fn setup_workspace_directory() -> Result<PathBuf> {
fn setup_workspace_directory(machine_mode: bool) -> Result<PathBuf> {
let workspace_dir = if let Ok(env_workspace) = std::env::var("G3_WORKSPACE") {
PathBuf::from(env_workspace)
} else {
@@ -1302,7 +1324,7 @@ fn setup_workspace_directory() -> Result<PathBuf> {
// Create the directory if it doesn't exist
if !workspace_dir.exists() {
std::fs::create_dir_all(&workspace_dir)?;
let output = SimpleOutput::new();
let output = SimpleOutput::new_with_mode(machine_mode);
output.print(&format!(
"📁 Created workspace directory: {}",
workspace_dir.display()

94
crates/g3-cli/src/machine_ui_writer.rs Normal file
View File

@@ -0,0 +1,94 @@
use g3_core::ui_writer::UiWriter;
use std::io::{self, Write};
/// Machine-mode implementation of UiWriter that prints plain, unformatted output
/// This is designed for programmatic consumption and outputs everything verbatim
pub struct MachineUiWriter;
impl MachineUiWriter {
pub fn new() -> Self {
Self
}
}
impl UiWriter for MachineUiWriter {
fn print(&self, message: &str) {
print!("{}", message);
}
fn println(&self, message: &str) {
println!("{}", message);
}
fn print_inline(&self, message: &str) {
print!("{}", message);
let _ = io::stdout().flush();
}
fn print_system_prompt(&self, prompt: &str) {
println!("SYSTEM_PROMPT:");
println!("{}", prompt);
println!("END_SYSTEM_PROMPT");
println!();
}
fn print_context_status(&self, message: &str) {
println!("CONTEXT_STATUS: {}", message);
}
fn print_context_thinning(&self, message: &str) {
println!("CONTEXT_THINNING: {}", message);
}
fn print_tool_header(&self, tool_name: &str) {
println!("TOOL_CALL: {}", tool_name);
}
fn print_tool_arg(&self, key: &str, value: &str) {
println!("TOOL_ARG: {} = {}", key, value);
}
fn print_tool_output_header(&self) {
println!("TOOL_OUTPUT:");
}
fn update_tool_output_line(&self, line: &str) {
println!("{}", line);
}
fn print_tool_output_line(&self, line: &str) {
println!("{}", line);
}
fn print_tool_output_summary(&self, count: usize) {
println!("TOOL_OUTPUT_LINES: {}", count);
}
fn print_tool_timing(&self, duration_str: &str) {
println!("TOOL_DURATION: {}", duration_str);
println!("END_TOOL_OUTPUT");
println!();
}
fn print_agent_prompt(&self) {
println!("AGENT_RESPONSE:");
let _ = io::stdout().flush();
}
fn print_agent_response(&self, content: &str) {
print!("{}", content);
let _ = io::stdout().flush();
}
fn notify_sse_received(&self) {
// No-op for machine mode
}
fn flush(&self) {
let _ = io::stdout().flush();
}
fn wants_full_output(&self) -> bool {
true // Machine mode wants complete, untruncated output
}
}

32
crates/g3-cli/src/simple_output.rs Normal file
View File

@@ -0,0 +1,32 @@
/// Simple output helper for printing messages
pub struct SimpleOutput {
machine_mode: bool,
}
impl SimpleOutput {
pub fn new() -> Self {
SimpleOutput { machine_mode: false }
}
pub fn new_with_mode(machine_mode: bool) -> Self {
SimpleOutput { machine_mode }
}
pub fn print(&self, message: &str) {
if !self.machine_mode {
println!("{}", message);
}
}
pub fn print_smart(&self, message: &str) {
if !self.machine_mode {
println!("{}", message);
}
}
}
impl Default for SimpleOutput {
fn default() -> Self {
Self::new()
}
}

240
crates/g3-cli/src/ui_writer_impl.rs
View File

@@ -1,8 +1,6 @@
use crate::retro_tui::RetroTui;
use g3_core::ui_writer::UiWriter;
use std::io::{self, Write};
use std::sync::Mutex;
use std::time::Instant;
/// Console implementation of UiWriter that prints to stdout
pub struct ConsoleUiWriter {
@@ -347,241 +345,3 @@ impl UiWriter for ConsoleUiWriter {
}
}
/// RetroTui implementation of UiWriter that sends output to the TUI
pub struct RetroTuiWriter {
tui: RetroTui,
current_tool_name: Mutex<Option<String>>,
current_tool_output: Mutex<Vec<String>>,
current_tool_start: Mutex<Option<Instant>>,
current_tool_caption: Mutex<String>,
}
impl RetroTuiWriter {
pub fn new(tui: RetroTui) -> Self {
Self {
tui,
current_tool_name: Mutex::new(None),
current_tool_output: Mutex::new(Vec::new()),
current_tool_start: Mutex::new(None),
current_tool_caption: Mutex::new(String::new()),
}
}
}
impl UiWriter for RetroTuiWriter {
fn print(&self, message: &str) {
self.tui.output(message);
}
fn println(&self, message: &str) {
self.tui.output(message);
}
fn print_inline(&self, message: &str) {
// For inline printing, we'll just append to the output
self.tui.output(message);
}
fn print_system_prompt(&self, prompt: &str) {
self.tui.output("🔍 System Prompt:");
self.tui.output("================");
for line in prompt.lines() {
self.tui.output(line);
}
self.tui.output("================");
self.tui.output("");
}
fn print_context_status(&self, message: &str) {
self.tui.output(message);
}
fn print_context_thinning(&self, message: &str) {
// For TUI, we'll use a highlighted output with special formatting
// The TUI will handle the visual presentation
// Add visual separators and emphasis
self.tui.output("");
self.tui.output("═══════════════════════════════════════════════════════════");
self.tui.output(&format!("✨ {} ✨", message));
self.tui.output(" └─ Context optimized successfully");
self.tui.output("═══════════════════════════════════════════════════════════");
self.tui.output("");
}
fn print_tool_header(&self, tool_name: &str) {
// Start collecting tool output
*self.current_tool_start.lock().unwrap() = Some(Instant::now());
*self.current_tool_name.lock().unwrap() = Some(tool_name.to_string());
self.current_tool_output.lock().unwrap().clear();
self.current_tool_output
.lock()
.unwrap()
.push(format!("Tool: {}", tool_name));
// Initialize caption
*self.current_tool_caption.lock().unwrap() = String::new();
}
fn print_tool_arg(&self, key: &str, value: &str) {
// Filter out any keys that look like they might be agent message content
// (e.g., keys that are suspiciously long or contain message-like content)
let is_valid_arg_key = key.len() < 50
&& !key.contains('\n')
&& !key.contains("I'll")
&& !key.contains("Let me")
&& !key.contains("Here's")
&& !key.contains("I can");
if is_valid_arg_key {
self.current_tool_output
.lock()
.unwrap()
.push(format!("{}: {}", key, value));
}
// Build caption from first argument (usually the most important one)
let mut caption = self.current_tool_caption.lock().unwrap();
if caption.is_empty() && (key == "file_path" || key == "command" || key == "path") {
// Truncate long values for the caption
let truncated = if value.len() > 50 {
// Use char_indices to safely truncate at character boundary
let truncate_at = value.char_indices()
.nth(47)
.map(|(i, _)| i)
.unwrap_or(value.len());
format!("{}...", &value[..truncate_at])
} else {
value.to_string()
};
// Add range information for read_file tool calls
let tool_name = self.current_tool_name.lock().unwrap();
let range_suffix = if tool_name.as_ref().is_some_and(|name| name == "read_file") {
// We need to check if start/end args will be provided - for now just check if this is a partial read
// This is a simplified approach since we're building the caption incrementally
String::new() // We'll handle this in print_tool_output_header instead
} else {
String::new()
};
*caption = format!("{}{}", truncated, range_suffix);
}
}
fn print_tool_output_header(&self) {
// This is called right before tool execution starts
// Send the initial tool header to the TUI now
if let Some(tool_name) = self.current_tool_name.lock().unwrap().as_ref() {
let mut caption = self.current_tool_caption.lock().unwrap().clone();
// Add range information for read_file tool calls
if tool_name == "read_file" {
// Check the tool output for start/end parameters
let output = self.current_tool_output.lock().unwrap();
let has_start = output.iter().any(|line| line.starts_with("start:"));
let has_end = output.iter().any(|line| line.starts_with("end:"));
if has_start || has_end {
let start_val = output.iter().find(|line| line.starts_with("start:")).map(|line| line.split(':').nth(1).unwrap_or("0").trim()).unwrap_or("0");
let end_val = output.iter().find(|line| line.starts_with("end:")).map(|line| line.split(':').nth(1).unwrap_or("end").trim()).unwrap_or("end");
caption = format!("{} [{}..{}]", caption, start_val, end_val);
}
}
// Send the tool output with initial header
self.tui.tool_output(tool_name, &caption, "");
}
self.current_tool_output.lock().unwrap().push(String::new());
self.current_tool_output
.lock()
.unwrap()
.push("Output:".to_string());
}
fn update_tool_output_line(&self, line: &str) {
// For retro mode, we'll just add to the output buffer
self.current_tool_output
.lock()
.unwrap()
.push(line.to_string());
}
fn print_tool_output_line(&self, line: &str) {
self.current_tool_output
.lock()
.unwrap()
.push(line.to_string());
}
fn print_tool_output_summary(&self, hidden_count: usize) {
self.current_tool_output.lock().unwrap().push(format!(
"... ({} more line{})",
hidden_count,
if hidden_count == 1 { "" } else { "s" }
));
}
fn print_tool_timing(&self, duration_str: &str) {
self.current_tool_output
.lock()
.unwrap()
.push(format!("⚡️ {}", duration_str));
// Calculate the actual duration
let duration_ms = if let Some(start) = *self.current_tool_start.lock().unwrap() {
start.elapsed().as_millis()
} else {
0
};
// Get the tool name and caption
if let Some(tool_name) = self.current_tool_name.lock().unwrap().as_ref() {
let content = self.current_tool_output.lock().unwrap().join("\n");
let caption = self.current_tool_caption.lock().unwrap().clone();
let caption = if caption.is_empty() {
"Completed".to_string()
} else {
caption
};
// Update the tool detail panel with the complete output without adding a new header
// This keeps the original header in place to be updated by tool_complete
self.tui.update_tool_detail(tool_name, &content);
// Determine success based on whether there's an error in the output
// This is a simple heuristic - you might want to make this more sophisticated
let success = !content.contains("error")
&& !content.contains("Error")
&& !content.contains("ERROR");
// Send the completion status to update the header
self.tui
.tool_complete(tool_name, success, duration_ms, &caption);
}
// Clear the buffers
*self.current_tool_name.lock().unwrap() = None;
self.current_tool_output.lock().unwrap().clear();
*self.current_tool_start.lock().unwrap() = None;
*self.current_tool_caption.lock().unwrap() = String::new();
}
fn print_agent_prompt(&self) {
self.tui.output("\n💬 ");
}
fn print_agent_response(&self, content: &str) {
self.tui.output(content);
}
fn notify_sse_received(&self) {
// Notify the TUI that an SSE was received
self.tui.sse_received();
}
fn flush(&self) {
// No-op for TUI since it handles its own rendering
}
}

4
crates/g3-computer-control/examples/list_windows.rs
View File

@@ -47,8 +47,8 @@ fn main() {
"".to_string()
};
// Filter for iTerm or show all
if owner.contains("iTerm") || owner.contains("Terminal") {
// Show all windows
if !owner.is_empty() {
println!("{:<10} {:<25} {}", window_id, owner, title);
}
}

323
crates/g3-computer-control/src/platform/macos.rs
View File

@@ -64,7 +64,8 @@ impl ComputerController for MacOSController {
let array = CFArray::<CFDictionary>::wrap_under_create_rule(window_list);
let count = array.len();
let mut found_window_id: Option<u32> = None;
let mut found_window_id: Option<(u32, String)> = None; // (id, owner)
let app_name_lower = app_name.to_lowercase();
for i in 0..count {
let dict = array.get(i).unwrap();
@@ -78,15 +79,66 @@ impl ComputerController for MacOSController {
continue;
};
// Check if this is the app we're looking for
if owner.to_lowercase().contains(&app_name.to_lowercase()) || app_name.to_lowercase().contains(&owner.to_lowercase()) {
tracing::debug!("Checking window: owner='{}', looking for '{}'", owner, app_name);
let owner_lower = owner.to_lowercase();
// Normalize by removing spaces for exact matching
let app_name_normalized = app_name_lower.replace(" ", "");
let owner_normalized = owner_lower.replace(" ", "");
// ONLY accept exact matches (case-insensitive, with or without spaces)
// This prevents "Goose" from matching "GooseStudio"
let is_match = owner_lower == app_name_lower || owner_normalized == app_name_normalized;
if is_match {
// Get window ID
let window_id_key = CFString::from_static_string("kCGWindowNumber");
if let Some(value) = dict.find(window_id_key.to_void()) {
let num: core_foundation::number::CFNumber = TCFType::wrap_under_get_rule(*value as *const _);
if let Some(id) = num.to_i64() {
found_window_id = Some(id as u32);
break;
// Get window layer to filter out menu bar windows
let layer_key = CFString::from_static_string("kCGWindowLayer");
let layer: i32 = if let Some(value) = dict.find(layer_key.to_void()) {
let num: core_foundation::number::CFNumber = TCFType::wrap_under_get_rule(*value as *const _);
num.to_i32().unwrap_or(0)
} else {
0
};
// Get window bounds to verify it's a real window
let bounds_key = CFString::from_static_string("kCGWindowBounds");
let has_real_bounds = if let Some(value) = dict.find(bounds_key.to_void()) {
let bounds_dict: CFDictionary = TCFType::wrap_under_get_rule(*value as *const _);
let width_key = CFString::from_static_string("Width");
let height_key = CFString::from_static_string("Height");
if let (Some(w_val), Some(h_val)) = (
bounds_dict.find(width_key.to_void()),
bounds_dict.find(height_key.to_void()),
) {
let w_num: core_foundation::number::CFNumber = TCFType::wrap_under_get_rule(*w_val as *const _);
let h_num: core_foundation::number::CFNumber = TCFType::wrap_under_get_rule(*h_val as *const _);
let width = w_num.to_f64().unwrap_or(0.0);
let height = h_num.to_f64().unwrap_or(0.0);
// Real windows should be at least 100x100 pixels
width >= 100.0 && height >= 100.0
} else {
false
}
} else {
false
};
// Only accept windows that are:
// 1. At layer 0 (normal windows, not menu bar)
// 2. Have real bounds (width and height >= 100)
if layer == 0 && has_real_bounds {
tracing::info!("Found valid window: ID {} for app '{}' (layer={}, bounds valid)", id, owner, layer);
found_window_id = Some((id as u32, owner.clone()));
break;
} else {
tracing::debug!("Skipping window ID {} for '{}': layer={}, has_real_bounds={}", id, owner, layer, has_real_bounds);
}
}
}
}
@@ -95,9 +147,10 @@ impl ComputerController for MacOSController {
found_window_id
};
let cg_window_id = cg_window_id.ok_or_else(|| {
let (cg_window_id, matched_owner) = cg_window_id.ok_or_else(|| {
anyhow::anyhow!("Could not find window for application '{}'. Use list_windows to see available windows.", app_name)
})?;
tracing::info!("Taking screenshot of window ID {} for app '{}'", cg_window_id, matched_owner);
// Use screencapture with the window ID for now
// TODO: Implement direct CGWindowListCreateImage approach with proper image saving
@@ -151,12 +204,18 @@ impl ComputerController for MacOSController {
async fn find_text_in_app(&self, app_name: &str, search_text: &str) -> Result<Option<TextLocation>> {
// Take screenshot of specific app window
let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
let temp_path = format!("{}/Desktop/g3_find_text_{}_{}.png", home, app_name, uuid::Uuid::new_v4());
let temp_path = format!("{}/tmp/g3_find_text_{}_{}.png", home, app_name, uuid::Uuid::new_v4());
self.take_screenshot(&temp_path, None, Some(app_name)).await?;
// Get screenshot dimensions before we delete it
let screenshot_dims = get_image_dimensions(&temp_path)?;
// Extract all text with locations
let locations = self.extract_text_with_locations(&temp_path).await?;
// Get window bounds to calculate coordinate transformation
let window_bounds = self.get_window_bounds(app_name)?;
// Clean up temp file
let _ = std::fs::remove_file(&temp_path);
@@ -164,7 +223,13 @@ impl ComputerController for MacOSController {
let search_lower = search_text.to_lowercase();
for location in locations {
if location.text.to_lowercase().contains(&search_lower) {
return Ok(Some(location));
// Transform coordinates from screenshot space to screen space
let transformed = transform_screenshot_to_screen_coords(
location,
window_bounds,
screenshot_dims,
);
return Ok(Some(transformed));
}
}
@@ -195,44 +260,7 @@ impl ComputerController for MacOSController {
Ok(())
}
fn click_at(&self, x: i32, y: i32, app_name: Option<&str>) -> Result<()> {
// If app_name is provided, get window position and offset coordinates
let (global_x, global_y) = if let Some(app) = app_name {
// Get window position using AppleScript
let script = format!(
r#"tell application "{}" to get bounds of window 1"#,
app
);
let output = std::process::Command::new("osascript")
.arg("-e")
.arg(&script)
.output()?;
if output.status.success() {
let bounds_str = String::from_utf8_lossy(&output.stdout);
// Parse bounds: "x1, y1, x2, y2"
let parts: Vec<&str> = bounds_str.trim().split(", ").collect();
if parts.len() >= 2 {
if let (Ok(window_x), Ok(window_y)) = (
parts[0].trim().parse::<i32>(),
parts[1].trim().parse::<i32>(),
) {
// Offset relative coordinates by window position
(x + window_x, y + window_y)
} else {
(x, y) // Fallback to absolute coordinates
}
} else {
(x, y) // Fallback to absolute coordinates
}
} else {
(x, y) // Fallback to absolute coordinates
}
} else {
(x, y) // No app name, use absolute coordinates
};
fn click_at(&self, x: i32, y: i32, _app_name: Option<&str>) -> Result<()> {
use core_graphics::event::{
CGEvent, CGEventTapLocation, CGEventType, CGMouseButton,
};
@@ -240,12 +268,27 @@ impl ComputerController for MacOSController {
CGEventSource, CGEventSourceStateID,
};
use core_graphics::geometry::CGPoint;
use core_graphics::display::CGDisplay;
// IMPORTANT: Coordinates passed here are in NSScreen/CGWindowListCopyWindowInfo space
// (Y=0 at BOTTOM, increases UPWARD)
// But CGEvent uses a different coordinate system (Y=0 at TOP, increases DOWNWARD)
// We need to convert: CGEvent.y = screenHeight - NSScreen.y
let screen_height = CGDisplay::main().pixels_high() as i32;
let cgevent_x = x;
let cgevent_y = screen_height - y;
tracing::debug!("click_at: NSScreen coords ({}, {}) -> CGEvent coords ({}, {}) [screen_height={}]",
x, y, cgevent_x, cgevent_y, screen_height);
let (global_x, global_y) = (cgevent_x, cgevent_y);
let point = CGPoint::new(global_x as f64, global_y as f64);
let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState)
.ok().context("Failed to create event source")?;
let point = CGPoint::new(global_x as f64, global_y as f64);
// Move mouse to position first
let move_event = CGEvent::new_mouse_event(
source.clone(),
@@ -279,4 +322,186 @@ impl ComputerController for MacOSController {
Ok(())
}
}
}
impl MacOSController {
/// Get window bounds for an application (helper method)
fn get_window_bounds(&self, app_name: &str) -> Result<(i32, i32, i32, i32)> {
unsafe {
let window_list = CGWindowListCopyWindowInfo(
kCGWindowListOptionOnScreenOnly,
kCGNullWindowID
);
let array = CFArray::<CFDictionary>::wrap_under_create_rule(window_list);
let count = array.len();
let app_name_lower = app_name.to_lowercase();
for i in 0..count {
let dict = array.get(i).unwrap();
// Get owner name
let owner_key = CFString::from_static_string("kCGWindowOwnerName");
let owner: String = if let Some(value) = dict.find(owner_key.to_void()) {
let s: CFString = TCFType::wrap_under_get_rule(*value as *const _);
s.to_string()
} else {
continue;
};
let owner_lower = owner.to_lowercase();
// Normalize by removing spaces for exact matching
let app_name_normalized = app_name_lower.replace(" ", "");
let owner_normalized = owner_lower.replace(" ", "");
// ONLY accept exact matches (case-insensitive, with or without spaces)
// This prevents "Goose" from matching "GooseStudio"
let is_match = owner_lower == app_name_lower || owner_normalized == app_name_normalized;
if is_match {
// Get window layer to filter out menu bar windows
let layer_key = CFString::from_static_string("kCGWindowLayer");
let layer: i32 = if let Some(value) = dict.find(layer_key.to_void()) {
let num: core_foundation::number::CFNumber = TCFType::wrap_under_get_rule(*value as *const _);
num.to_i32().unwrap_or(0)
} else {
0
};
// Skip menu bar windows (layer >= 20)
if layer >= 20 {
tracing::debug!("Skipping window for '{}' at layer {} (menu bar)", owner, layer);
continue;
}
// Get window bounds to verify it's a real window
let bounds_key = CFString::from_static_string("kCGWindowBounds");
if let Some(value) = dict.find(bounds_key.to_void()) {
let bounds_dict: CFDictionary = TCFType::wrap_under_get_rule(*value as *const _);
let x_key = CFString::from_static_string("X");
let y_key = CFString::from_static_string("Y");
let width_key = CFString::from_static_string("Width");
let height_key = CFString::from_static_string("Height");
if let (Some(x_val), Some(y_val), Some(w_val), Some(h_val)) = (
bounds_dict.find(x_key.to_void()),
bounds_dict.find(y_key.to_void()),
bounds_dict.find(width_key.to_void()),
bounds_dict.find(height_key.to_void()),
) {
let x_num: core_foundation::number::CFNumber = TCFType::wrap_under_get_rule(*x_val as *const _);
let y_num: core_foundation::number::CFNumber = TCFType::wrap_under_get_rule(*y_val as *const _);
let w_num: core_foundation::number::CFNumber = TCFType::wrap_under_get_rule(*w_val as *const _);
let h_num: core_foundation::number::CFNumber = TCFType::wrap_under_get_rule(*h_val as *const _);
let x: i32 = x_num.to_i64().unwrap_or(0) as i32;
let y: i32 = y_num.to_i64().unwrap_or(0) as i32;
let w: i32 = w_num.to_i64().unwrap_or(0) as i32;
let h: i32 = h_num.to_i64().unwrap_or(0) as i32;
// Only accept windows with real bounds (>= 100x100 pixels)
if w >= 100 && h >= 100 {
tracing::info!("Found valid window bounds for '{}': x={}, y={}, w={}, h={} (layer={})", owner, x, y, w, h, layer);
return Ok((x, y, w, h));
} else {
tracing::debug!("Skipping window for '{}': too small ({}x{})", owner, w, h);
continue;
}
} else {
continue;
}
}
}
}
}
Err(anyhow::anyhow!("Could not find window bounds for '{}'", app_name))
}
}
/// Get image dimensions from a PNG file
fn get_image_dimensions(path: &str) -> Result<(i32, i32)> {
use std::fs::File;
use std::io::Read;
let mut file = File::open(path)?;
let mut buffer = vec![0u8; 24];
file.read_exact(&mut buffer)?;
// PNG signature check
if &buffer[0..8] != b"\x89PNG\r\n\x1a\n" {
anyhow::bail!("Not a valid PNG file");
}
// Read IHDR chunk (width and height are at bytes 16-23)
let width = u32::from_be_bytes([buffer[16], buffer[17], buffer[18], buffer[19]]) as i32;
let height = u32::from_be_bytes([buffer[20], buffer[21], buffer[22], buffer[23]]) as i32;
Ok((width, height))
}
/// Transform coordinates from screenshot space to screen space
///
/// The screenshot is taken of a window, and Vision OCR returns coordinates
/// relative to the screenshot image. We need to transform these to actual
/// screen coordinates for clicking.
///
/// On Retina displays, screenshots are taken at 2x resolution, so we need
/// to account for this scaling factor.
fn transform_screenshot_to_screen_coords(
location: TextLocation,
window_bounds: (i32, i32, i32, i32), // (x, y, width, height) in screen space
screenshot_dims: (i32, i32), // (width, height) in pixels
) -> TextLocation {
let (win_x, win_y, win_width, win_height) = window_bounds;
let (screenshot_width, screenshot_height) = screenshot_dims;
// Calculate scale factors
// On Retina displays, screenshot is typically 2x the window size
let scale_x = win_width as f64 / screenshot_width as f64;
let scale_y = win_height as f64 / screenshot_height as f64;
tracing::debug!("Transform: screenshot={}x{}, window={}x{} at ({},{}), scale=({:.2},{:.2})",
screenshot_width, screenshot_height, win_width, win_height, win_x, win_y, scale_x, scale_y);
// Transform coordinates from image space to screen space
// IMPORTANT: macOS screen coordinates have origin at BOTTOM-LEFT (Y increases upward)
// Image coordinates have origin at TOP-LEFT (Y increases downward)
// win_y is the BOTTOM of the window in screen coordinates
// So we need to: (win_y + win_height) to get window TOP, then subtract screenshot_y
let window_top_y = win_y + win_height;
tracing::debug!("[transform] Input location in image space: x={}, y={}, width={}, height={}",
location.x, location.y, location.width, location.height);
tracing::debug!("[transform] Scale factors: scale_x={:.4}, scale_y={:.4}", scale_x, scale_y);
let transformed_x = win_x + (location.x as f64 * scale_x) as i32;
let transformed_y = window_top_y - (location.y as f64 * scale_y) as i32;
let transformed_width = (location.width as f64 * scale_x) as i32;
let transformed_height = (location.height as f64 * scale_y) as i32;
tracing::debug!("[transform] Calculation details:");
tracing::debug!(" - transformed_x = {} + ({} * {:.4}) = {} + {:.2} = {}", win_x, location.x, scale_x, win_x, location.x as f64 * scale_x, transformed_x);
tracing::debug!(" - transformed_width = ({} * {:.4}) = {:.2} -> {}", location.width, scale_x, location.width as f64 * scale_x, transformed_width);
tracing::debug!(" - transformed_height = ({} * {:.4}) = {:.2} -> {}", location.height, scale_y, location.height as f64 * scale_y, transformed_height);
tracing::debug!("Transformed location: screenshot=({},{}) {}x{} -> screen=({},{}) {}x{}",
location.x, location.y, location.width, location.height,
transformed_x, transformed_y, transformed_width, transformed_height);
TextLocation {
text: location.text,
x: transformed_x,
y: transformed_y,
width: transformed_width,
height: transformed_height,
confidence: location.confidence,
}
}
#[path = "macos_window_matching_test.rs"]
#[cfg(test)]
mod tests;

45
crates/g3-computer-control/src/platform/macos_window_matching_test.rs Normal file
View File

@@ -0,0 +1,45 @@
#[cfg(test)]
mod window_matching_tests {
/// Test that window name matching handles spaces correctly
///
/// Issue: When a user requests a screenshot of "Goose Studio" but the actual
/// application name is "GooseStudio" (no space), the fuzzy matching should
/// still find the window.
///
/// The fix normalizes both names by removing spaces before comparing.
#[test]
fn test_space_normalization() {
let test_cases = vec![
// (user_input, actual_app_name, should_match)
("Goose Studio", "GooseStudio", true),
("GooseStudio", "Goose Studio", true),
("Visual Studio Code", "VisualStudioCode", true),
("Google Chrome", "Google Chrome", true),
("Safari", "Safari", true),
("iTerm", "iTerm2", true), // fuzzy match
("Code", "Visual Studio Code", true), // fuzzy match
];
for (user_input, app_name, should_match) in test_cases {
let user_lower = user_input.to_lowercase();
let app_lower = app_name.to_lowercase();
let user_normalized = user_lower.replace(" ", "");
let app_normalized = app_lower.replace(" ", "");
let is_exact = app_lower == user_lower || app_normalized == user_normalized;
let is_fuzzy = app_lower.contains(&user_lower)
|| user_lower.contains(&app_lower)
|| app_normalized.contains(&user_normalized)
|| user_normalized.contains(&app_normalized);
let matches = is_exact || is_fuzzy;
assert_eq!(
matches, should_match,
"Expected '{}' vs '{}' to match={}, but got match={}",
user_input, app_name, should_match, matches
);
}
}
}

435
crates/g3-core/src/lib.rs
View File

@@ -483,8 +483,8 @@ Format this as a detailed but concise summary that can be used to resume the con
if matches!(message.role, MessageRole::User) && message.content.starts_with("Tool result:") {
let content_len = message.content.len();
// Only thin if the content is greater than 1000 chars
if content_len > 1000 {
// Only thin if the content is greater than 500 chars
if content_len > 500 {
// Generate a unique filename based on timestamp and index
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
@@ -541,8 +541,8 @@ Format this as a detailed but concise summary that can be used to resume the con
.map(|s| (s.to_string(), s.len()));
if let Some((content_str, content_len)) = content_info {
// Only thin if content is greater than 1000 chars
if content_len > 1000 {
// Only thin if content is greater than 500 chars
if content_len > 500 {
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
@@ -574,8 +574,8 @@ Format this as a detailed but concise summary that can be used to resume the con
.map(|s| (s.to_string(), s.len()));
if let Some((diff_str, diff_len)) = diff_info {
// Only thin if diff is greater than 1000 chars
if diff_len > 1000 {
// Only thin if diff is greater than 500 chars
if diff_len > 500 {
let timestamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
@@ -2080,132 +2080,6 @@ Template:
"required": ["app_name"]
}),
},
Tool {
name: "macax_get_ui_tree".to_string(),
description: "Get the UI element hierarchy of an application as a tree structure".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"app_name": {
"type": "string",
"description": "Name of the application"
},
"max_depth": {
"type": "integer",
"description": "Maximum depth to traverse (default: 3)"
}
},
"required": ["app_name"]
}),
},
Tool {
name: "macax_find_elements".to_string(),
description: "Find UI elements in an application by role, title, or identifier. Use this to locate buttons, text fields, etc.".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"app_name": {
"type": "string",
"description": "Name of the application"
},
"role": {
"type": "string",
"description": "UI element role (e.g., 'button', 'text field', 'window')"
},
"title": {
"type": "string",
"description": "Element title or label to match"
},
"identifier": {
"type": "string",
"description": "Element identifier (accessibility identifier)"
}
},
"required": ["app_name"]
}),
},
Tool {
name: "macax_click".to_string(),
description: "Click a UI element in an application".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"app_name": {
"type": "string",
"description": "Name of the application"
},
"role": {
"type": "string",
"description": "UI element role (e.g., 'button')"
},
"title": {
"type": "string",
"description": "Element title or label"
},
"identifier": {
"type": "string",
"description": "Element identifier"
}
},
"required": ["app_name", "role"]
}),
},
Tool {
name: "macax_set_value".to_string(),
description: "Set the value of a UI element (e.g., type into a text field)".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"app_name": {
"type": "string",
"description": "Name of the application"
},
"role": {
"type": "string",
"description": "UI element role (e.g., 'text field')"
},
"value": {
"type": "string",
"description": "Value to set"
},
"title": {
"type": "string",
"description": "Element title or label"
},
"identifier": {
"type": "string",
"description": "Element identifier"
}
},
"required": ["app_name", "role", "value"]
}),
},
Tool {
name: "macax_get_value".to_string(),
description: "Get the value of a UI element (e.g., read text from a text field)".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"app_name": {
"type": "string",
"description": "Name of the application"
},
"role": {
"type": "string",
"description": "UI element role (e.g., 'text field')"
},
"title": {
"type": "string",
"description": "Element title or label"
},
"identifier": {
"type": "string",
"description": "Element identifier"
}
},
"required": ["app_name", "role"]
}),
},
Tool {
name: "macax_press_key".to_string(),
description: "Press a keyboard key or shortcut in an application (e.g., Cmd+S to save)".to_string(),
@@ -2253,21 +2127,6 @@ Template:
}),
});
// Add focus_element tool
tools.push(Tool {
name: "macax_focus_element".to_string(),
description: "Focus on a UI element (text field, text area, etc.) before typing".to_string(),
input_schema: json!({
"type": "object",
"properties": {
"app_name": {"type": "string", "description": "Name of the application"},
"role": {"type": "string", "description": "UI element role (e.g., 'text field', 'text area')"},
"title": {"type": "string", "description": "Element title or label (optional)"},
"identifier": {"type": "string", "description": "Element accessibility identifier (optional)"}
},
"required": ["app_name", "role"]
}),
});
}
// Add extract_text_with_boxes tool (requires macax flag)
@@ -2816,14 +2675,26 @@ Template:
// Display tool execution result with proper indentation
if tool_call.tool != "final_output" {
let output_lines: Vec<&str> = tool_result.lines().collect();
// Skip displaying output for shell tool since it was already streamed
let should_display_output = tool_call.tool != "shell";
let output_lines: Vec<&str> = if should_display_output {
tool_result.lines().collect()
} else { vec![] };
// Check if UI wants full output (machine mode) or truncated (human mode)
let wants_full = self.ui_writer.wants_full_output();
// Helper function to safely truncate strings at character boundaries
let truncate_line = |line: &str, max_width: usize| -> String {
let char_count = line.chars().count();
if char_count <= max_width {
let truncate_line = |line: &str, max_width: usize, truncate: bool| -> String {
if !truncate {
// Machine mode - return full line
line.to_string()
} else if line.chars().count() <= max_width {
// Human mode - line fits within limit
line.to_string()
} else {
// Human mode - truncate long line
let truncated: String = line
.chars()
.take(max_width.saturating_sub(3))
@@ -2838,25 +2709,26 @@ Template:
// For todo tools, show all lines without truncation
let is_todo_tool = tool_call.tool == "todo_read" || tool_call.tool == "todo_write";
let max_lines_to_show = if is_todo_tool { output_len } else { MAX_LINES };
let max_lines_to_show = if is_todo_tool || wants_full { output_len } else { MAX_LINES };
for (idx, line) in output_lines.iter().enumerate() {
if !is_todo_tool && idx >= max_lines_to_show {
if !is_todo_tool && !wants_full && idx >= max_lines_to_show {
break;
}
// Clip line to max width
let clipped_line = truncate_line(line, MAX_LINE_WIDTH);
let clipped_line = truncate_line(line, MAX_LINE_WIDTH, !wants_full);
self.ui_writer.update_tool_output_line(&clipped_line);
}
if !is_todo_tool && output_len > MAX_LINES {
if !is_todo_tool && !wants_full && output_len > MAX_LINES {
self.ui_writer.print_tool_output_summary(output_len);
}
}
// Check if this was a final_output tool call
if tool_call.tool == "final_output" {
full_response.push_str(final_display_content);
// Don't add final_display_content here - it was already added before tool execution
// Adding it again would duplicate the output
if let Some(summary) = tool_call.args.get("summary") {
if let Some(summary_str) = summary.as_str() {
full_response.push_str(&format!("\n\n{}", summary_str));
@@ -3320,13 +3192,16 @@ Template:
{
Ok(result) => {
if result.success {
Ok(if result.stdout.is_empty() {
"✅ Command executed successfully".to_string()
} else {
result.stdout.trim().to_string()
})
// Don't return stdout - it was already streamed to the UI
// Returning it would cause duplicate output
Ok("✅ Command executed successfully".to_string())
} else {
Ok(format!("❌ Command failed: {}", result.stderr.trim()))
// For errors, return stderr since it wasn't streamed
Ok(if result.stderr.is_empty() {
"❌ Command failed".to_string()
} else {
format!("❌ Command failed: {}", result.stderr.trim())
})
}
}
Err(e) => Ok(format!("❌ Execution error: {}", e)),
@@ -4323,168 +4198,6 @@ Template:
Err(e) => Ok(format!("❌ Failed to activate app: {}", e)),
}
}
"macax_get_ui_tree" => {
debug!("Processing macax_get_ui_tree tool call");
if !self.config.macax.enabled {
return Ok("❌ macOS Accessibility is not enabled. Use --macax flag to enable.".to_string());
}
let app_name = match tool_call.args.get("app_name").and_then(|v| v.as_str()) {
Some(n) => n,
None => return Ok("❌ Missing app_name argument".to_string()),
};
let max_depth = tool_call.args.get("max_depth")
.and_then(|v| v.as_u64())
.map(|n| n as usize)
.unwrap_or(3);
let controller_guard = self.macax_controller.read().await;
let controller = match controller_guard.as_ref() {
Some(c) => c,
None => return Ok("❌ macOS Accessibility controller not initialized.".to_string()),
};
match controller.get_ui_tree(app_name, max_depth) {
Ok(tree) => Ok(tree),
Err(e) => Ok(format!("❌ Failed to get UI tree: {}", e)),
}
}
"macax_find_elements" => {
debug!("Processing macax_find_elements tool call");
if !self.config.macax.enabled {
return Ok("❌ macOS Accessibility is not enabled. Use --macax flag to enable.".to_string());
}
let app_name = match tool_call.args.get("app_name").and_then(|v| v.as_str()) {
Some(n) => n,
None => return Ok("❌ Missing app_name argument".to_string()),
};
let role = tool_call.args.get("role").and_then(|v| v.as_str());
let title = tool_call.args.get("title").and_then(|v| v.as_str());
let identifier = tool_call.args.get("identifier").and_then(|v| v.as_str());
let controller_guard = self.macax_controller.read().await;
let controller = match controller_guard.as_ref() {
Some(c) => c,
None => return Ok("❌ macOS Accessibility controller not initialized.".to_string()),
};
match controller.find_elements(app_name, role, title, identifier) {
Ok(elements) => {
if elements.is_empty() {
Ok("No elements found matching criteria".to_string())
} else {
let element_strs: Vec<String> = elements.iter()
.map(|e| e.to_string())
.collect();
Ok(format!("Found {} element(s):\n{}", elements.len(), element_strs.join("\n")))
}
}
Err(e) => Ok(format!("❌ Failed to find elements: {}", e)),
}
}
"macax_click" => {
debug!("Processing macax_click tool call");
if !self.config.macax.enabled {
return Ok("❌ macOS Accessibility is not enabled. Use --macax flag to enable.".to_string());
}
let app_name = match tool_call.args.get("app_name").and_then(|v| v.as_str()) {
Some(n) => n,
None => return Ok("❌ Missing app_name argument".to_string()),
};
let role = match tool_call.args.get("role").and_then(|v| v.as_str()) {
Some(r) => r,
None => return Ok("❌ Missing role argument".to_string()),
};
let title = tool_call.args.get("title").and_then(|v| v.as_str());
let identifier = tool_call.args.get("identifier").and_then(|v| v.as_str());
let controller_guard = self.macax_controller.read().await;
let controller = match controller_guard.as_ref() {
Some(c) => c,
None => return Ok("❌ macOS Accessibility controller not initialized.".to_string()),
};
match controller.click_element(app_name, role, title, identifier) {
Ok(_) => Ok(format!("✅ Clicked {} element", role)),
Err(e) => Ok(format!("❌ Failed to click element: {}", e)),
}
}
"macax_set_value" => {
debug!("Processing macax_set_value tool call");
if !self.config.macax.enabled {
return Ok("❌ macOS Accessibility is not enabled. Use --macax flag to enable.".to_string());
}
let app_name = match tool_call.args.get("app_name").and_then(|v| v.as_str()) {
Some(n) => n,
None => return Ok("❌ Missing app_name argument".to_string()),
};
let role = match tool_call.args.get("role").and_then(|v| v.as_str()) {
Some(r) => r,
None => return Ok("❌ Missing role argument".to_string()),
};
let value = match tool_call.args.get("value").and_then(|v| v.as_str()) {
Some(v) => v,
None => return Ok("❌ Missing value argument".to_string()),
};
let title = tool_call.args.get("title").and_then(|v| v.as_str());
let identifier = tool_call.args.get("identifier").and_then(|v| v.as_str());
let controller_guard = self.macax_controller.read().await;
let controller = match controller_guard.as_ref() {
Some(c) => c,
None => return Ok("❌ macOS Accessibility controller not initialized.".to_string()),
};
match controller.set_value(app_name, role, value, title, identifier) {
Ok(_) => Ok(format!("✅ Set value of {} element to: {}", role, value)),
Err(e) => Ok(format!("❌ Failed to set value: {}", e)),
}
}
"macax_get_value" => {
debug!("Processing macax_get_value tool call");
if !self.config.macax.enabled {
return Ok("❌ macOS Accessibility is not enabled. Use --macax flag to enable.".to_string());
}
let app_name = match tool_call.args.get("app_name").and_then(|v| v.as_str()) {
Some(n) => n,
None => return Ok("❌ Missing app_name argument".to_string()),
};
let role = match tool_call.args.get("role").and_then(|v| v.as_str()) {
Some(r) => r,
None => return Ok("❌ Missing role argument".to_string()),
};
let title = tool_call.args.get("title").and_then(|v| v.as_str());
let identifier = tool_call.args.get("identifier").and_then(|v| v.as_str());
let controller_guard = self.macax_controller.read().await;
let controller = match controller_guard.as_ref() {
Some(c) => c,
None => return Ok("❌ macOS Accessibility controller not initialized.".to_string()),
};
match controller.get_value(app_name, role, title, identifier) {
Ok(value) => Ok(format!("Value: {}", value)),
Err(e) => Ok(format!("❌ Failed to get value: {}", e)),
}
}
"macax_press_key" => {
debug!("Processing macax_press_key tool call");
@@ -4555,37 +4268,6 @@ Template:
Err(e) => Ok(format!("❌ Failed to type text: {}", e)),
}
}
"macax_focus_element" => {
debug!("Processing macax_focus_element tool call");
if !self.config.macax.enabled {
return Ok("❌ macOS Accessibility is not enabled. Use --macax flag to enable.".to_string());
}
let app_name = match tool_call.args.get("app_name").and_then(|v| v.as_str()) {
Some(n) => n,
None => return Ok("❌ Missing app_name argument".to_string()),
};
let role = match tool_call.args.get("role").and_then(|v| v.as_str()) {
Some(r) => r,
None => return Ok("❌ Missing role argument".to_string()),
};
let title = tool_call.args.get("title").and_then(|v| v.as_str());
let identifier = tool_call.args.get("identifier").and_then(|v| v.as_str());
let controller_guard = self.macax_controller.read().await;
let controller = match controller_guard.as_ref() {
Some(c) => c,
None => return Ok("❌ macOS Accessibility controller not initialized.".to_string()),
};
match controller.focus_element(app_name, role, title, identifier) {
Ok(_) => Ok(format!("✅ Focused {} element in {}", role, app_name)),
Err(e) => Ok(format!("❌ Failed to focus element: {}", e)),
}
}
"vision_find_text" => {
debug!("Processing vision_find_text tool call");
@@ -4628,11 +4310,34 @@ Template:
match controller.find_text_in_app(app_name, text).await {
Ok(Some(location)) => {
// Click on center of text
let center_x = location.x + location.width / 2;
let center_y = location.y + location.height / 2;
// IMPORTANT: location coordinates are in NSScreen space (Y=0 at BOTTOM, increases UPWARD)
// location.x is the LEFT edge of the bounding box
// location.y is the TOP edge of the bounding box (highest Y value in NSScreen space)
// location.width and location.height are already scaled to screen space
// To get center: we need to add half the SCALED width and subtract half the SCALED height
match controller.click_at(center_x, center_y, Some(app_name)) {
Ok(_) => Ok(format!("✅ Clicked on '{}' in {} at ({}, {})", text, app_name, center_x, center_y)),
if location.width == 0 || location.height == 0 {
return Ok(format!("❌ Invalid bounding box dimensions: width={}, height={}", location.width, location.height));
}
debug!("[vision_click_text] Location from find_text_in_app: x={}, y={}, width={}, height={}, text='{}'",
location.x, location.y, location.width, location.height, location.text);
// Calculate center using the SCALED dimensions
// X: Use right edge instead of center (Vision OCR bounding box seems offset)
// This gives us: left edge + full width = right edge
// Y: top edge - half of scaled height (subtract because Y increases upward)
let click_x = location.x + location.width; // Right edge
let half_height = location.height / 2;
let click_y = location.y - half_height;
debug!("[vision_click_text] Click position calculation: x={} + {} = {} (right edge), y={} - {} = {}",
location.x, location.width, click_x, location.y, half_height, click_y);
debug!("[vision_click_text] This means: left_edge={}, center={}, right_edge={}",
location.x, click_x, location.x + location.width);
match controller.click_at(click_x, click_y, Some(app_name)) {
Ok(_) => Ok(format!("✅ Clicked on '{}' in {} at ({}, {})", text, app_name, click_x, click_y)),
Err(e) => Ok(format!("❌ Failed to click: {}", e)),
}
}
@@ -4709,13 +4414,15 @@ Template:
match controller.find_text_in_app(app_name, text).await {
Ok(Some(location)) => {
// Calculate click position based on direction
// location.x is LEFT edge, location.y is TOP edge (in NSScreen space)
let (click_x, click_y) = match direction {
"right" => (location.x + location.width + distance, location.y + location.height / 2),
"below" => (location.x + location.width / 2, location.y + location.height + distance),
"left" => (location.x - distance, location.y + location.height / 2),
"above" => (location.x + location.width / 2, location.y - distance),
_ => (location.x + location.width + distance, location.y + location.height / 2),
"right" => (location.x + location.width + distance, location.y - (location.height / 2)),
"below" => (location.x + (location.width / 2), location.y - location.height - distance),
"left" => (location.x - distance, location.y - (location.height / 2)),
"above" => (location.x + (location.width / 2), location.y + distance),
_ => (location.x + location.width + distance, location.y - (location.height / 2)),
};
debug!("[vision_click_near_text] Clicking {} of text at ({}, {})", direction, click_x, click_y);
match controller.click_at(click_x, click_y, Some(app_name)) {
Ok(_) => Ok(format!(

5
crates/g3-core/src/ui_writer.rs
View File

@@ -52,6 +52,10 @@ pub trait UiWriter: Send + Sync {
/// Flush any buffered output
fn flush(&self);
/// Returns true if this UI writer wants full, untruncated output
/// Default is false (truncate for human readability)
fn wants_full_output(&self) -> bool { false }
}
/// A no-op implementation for when UI output is not needed
@@ -75,4 +79,5 @@ impl UiWriter for NullUiWriter {
fn print_agent_response(&self, _content: &str) {}
fn notify_sse_received(&self) {}
fn flush(&self) {}
fn wants_full_output(&self) -> bool { false }
}

48
crates/g3-execution/src/lib.rs
View File

@@ -166,6 +166,31 @@ impl CodeExecutor {
/// Execute Bash code
async fn execute_bash(&self, code: &str) -> Result<ExecutionResult> {
// Check if this is a detached/daemon command that should run independently
let is_detached = code.trim_start().starts_with("setsid ")
|| code.trim_start().starts_with("nohup ")
|| code.contains(" disown")
|| (code.contains(" &") && (code.contains("nohup") || code.contains("setsid")));
if is_detached {
// For detached commands, just spawn and return immediately
use std::process::Stdio;
Command::new("bash")
.arg("-c")
.arg(code)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()?;
return Ok(ExecutionResult {
stdout: "✅ Command launched in background (detached process)".to_string(),
stderr: String::new(),
exit_code: 0,
success: true,
});
}
let output = Command::new("bash")
.arg("-c")
.arg(code)
@@ -221,6 +246,29 @@ impl CodeExecutor {
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::Command as TokioCommand;
// Check if this is a detached/daemon command that should run independently
// Look for patterns like: setsid, nohup with &, or explicit backgrounding with disown
let is_detached = code.trim_start().starts_with("setsid ")
|| code.trim_start().starts_with("nohup ")
|| code.contains(" disown")
|| (code.contains(" &") && (code.contains("nohup") || code.contains("setsid")));
if is_detached {
// For detached commands, just spawn and return immediately
TokioCommand::new("bash")
.arg("-c")
.arg(code)
.spawn()?;
// Don't wait for the process - it's meant to run independently
return Ok(ExecutionResult {
stdout: "✅ Command launched in background (detached process)".to_string(),
stderr: String::new(),
exit_code: 0,
success: true,
});
}
let mut child = TokioCommand::new("bash")
.arg("-c")
.arg(code)