turn histogram from autonomous mode
This commit is contained in:
@@ -1,6 +1,104 @@
|
|||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
/// Extract coach feedback by reading from the coach agent's specific log file
|
/// Extract coach feedback by reading from the coach agent's specific log file
|
||||||
/// Uses the coach agent's session ID to find the exact log file
|
/// Uses the coach agent's session ID to find the exact log file
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct TurnMetrics {
|
||||||
|
turn_number: usize,
|
||||||
|
tokens_used: u32,
|
||||||
|
wall_clock_time: Duration,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Generate a histogram showing tokens used and wall clock time per turn
|
||||||
|
fn generate_turn_histogram(turn_metrics: &[TurnMetrics]) -> String {
|
||||||
|
if turn_metrics.is_empty() {
|
||||||
|
return " No turn data available".to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut histogram = String::new();
|
||||||
|
|
||||||
|
// Find max values for scaling
|
||||||
|
let max_tokens = turn_metrics.iter().map(|t| t.tokens_used).max().unwrap_or(1);
|
||||||
|
let max_time_ms = turn_metrics.iter()
|
||||||
|
.map(|t| t.wall_clock_time.as_millis() as u32)
|
||||||
|
.max()
|
||||||
|
.unwrap_or(1);
|
||||||
|
|
||||||
|
// Constants for histogram display
|
||||||
|
const MAX_BAR_WIDTH: usize = 40;
|
||||||
|
const TOKEN_CHAR: char = '█';
|
||||||
|
const TIME_CHAR: char = '▓';
|
||||||
|
|
||||||
|
histogram.push_str("\n📊 Per-Turn Performance Histogram:\n");
|
||||||
|
histogram.push_str(&format!(" {} = Tokens Used (max: {})\n", TOKEN_CHAR, max_tokens));
|
||||||
|
histogram.push_str(&format!(" {} = Wall Clock Time (max: {:.1}s)\n\n", TIME_CHAR, max_time_ms as f64 / 1000.0));
|
||||||
|
|
||||||
|
for metrics in turn_metrics {
|
||||||
|
let turn_time_ms = metrics.wall_clock_time.as_millis() as u32;
|
||||||
|
|
||||||
|
// Calculate bar lengths (proportional to max values)
|
||||||
|
let token_bar_len = if max_tokens > 0 {
|
||||||
|
((metrics.tokens_used as f64 / max_tokens as f64) * MAX_BAR_WIDTH as f64) as usize
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
|
||||||
|
let time_bar_len = if max_time_ms > 0 {
|
||||||
|
((turn_time_ms as f64 / max_time_ms as f64) * MAX_BAR_WIDTH as f64) as usize
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
|
||||||
|
// Format time duration
|
||||||
|
let time_str = if turn_time_ms < 1000 {
|
||||||
|
format!("{}ms", turn_time_ms)
|
||||||
|
} else if turn_time_ms < 60_000 {
|
||||||
|
format!("{:.1}s", turn_time_ms as f64 / 1000.0)
|
||||||
|
} else {
|
||||||
|
let minutes = turn_time_ms / 60_000;
|
||||||
|
let seconds = (turn_time_ms % 60_000) as f64 / 1000.0;
|
||||||
|
format!("{}m{:.1}s", minutes, seconds)
|
||||||
|
};
|
||||||
|
|
||||||
|
// Create the bars
|
||||||
|
let token_bar = TOKEN_CHAR.to_string().repeat(token_bar_len);
|
||||||
|
let time_bar = TIME_CHAR.to_string().repeat(time_bar_len);
|
||||||
|
|
||||||
|
// Add turn information
|
||||||
|
histogram.push_str(&format!(
|
||||||
|
" Turn {:2}: {:>6} tokens │{:<40}│\n",
|
||||||
|
metrics.turn_number,
|
||||||
|
metrics.tokens_used,
|
||||||
|
token_bar
|
||||||
|
));
|
||||||
|
histogram.push_str(&format!(
|
||||||
|
" {:>6} │{:<40}│\n",
|
||||||
|
time_str,
|
||||||
|
time_bar
|
||||||
|
));
|
||||||
|
|
||||||
|
// Add separator line between turns (except for last turn)
|
||||||
|
if metrics.turn_number != turn_metrics.last().unwrap().turn_number {
|
||||||
|
histogram.push_str(" ────────────┼────────────────────────────────────────┤\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add summary statistics
|
||||||
|
let total_tokens: u32 = turn_metrics.iter().map(|t| t.tokens_used).sum();
|
||||||
|
let total_time: Duration = turn_metrics.iter().map(|t| t.wall_clock_time).sum();
|
||||||
|
let avg_tokens = total_tokens as f64 / turn_metrics.len() as f64;
|
||||||
|
let avg_time_ms = total_time.as_millis() as f64 / turn_metrics.len() as f64;
|
||||||
|
|
||||||
|
histogram.push_str("\n📈 Summary Statistics:\n");
|
||||||
|
histogram.push_str(&format!(" • Total Tokens: {} across {} turns\n", total_tokens, turn_metrics.len()));
|
||||||
|
histogram.push_str(&format!(" • Average Tokens/Turn: {:.1}\n", avg_tokens));
|
||||||
|
histogram.push_str(&format!(" • Total Time: {:.1}s\n", total_time.as_secs_f64()));
|
||||||
|
histogram.push_str(&format!(" • Average Time/Turn: {:.1}s\n", avg_time_ms / 1000.0));
|
||||||
|
|
||||||
|
histogram
|
||||||
|
}
|
||||||
|
|
||||||
fn extract_coach_feedback_from_logs(_coach_result: &g3_core::TaskResult, coach_agent: &g3_core::Agent<ConsoleUiWriter>, output: &SimpleOutput) -> Result<String> {
|
fn extract_coach_feedback_from_logs(_coach_result: &g3_core::TaskResult, coach_agent: &g3_core::Agent<ConsoleUiWriter>, output: &SimpleOutput) -> Result<String> {
|
||||||
// CORRECT APPROACH: Get the session ID from the current coach agent
|
// CORRECT APPROACH: Get the session ID from the current coach agent
|
||||||
// and read its specific log file directly
|
// and read its specific log file directly
|
||||||
@@ -940,6 +1038,7 @@ async fn run_autonomous(
|
|||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let start_time = std::time::Instant::now();
|
let start_time = std::time::Instant::now();
|
||||||
let output = SimpleOutput::new();
|
let output = SimpleOutput::new();
|
||||||
|
let mut turn_metrics: Vec<TurnMetrics> = Vec::new();
|
||||||
|
|
||||||
output.print("🤖 G3 AI Coding Agent - Autonomous Mode");
|
output.print("🤖 G3 AI Coding Agent - Autonomous Mode");
|
||||||
output.print(&format!(
|
output.print(&format!(
|
||||||
@@ -986,6 +1085,8 @@ async fn run_autonomous(
|
|||||||
" • Usage Percentage: {:.1}%",
|
" • Usage Percentage: {:.1}%",
|
||||||
context_window.percentage_used()
|
context_window.percentage_used()
|
||||||
));
|
));
|
||||||
|
// Add per-turn histogram
|
||||||
|
output.print(&generate_turn_histogram(&turn_metrics));
|
||||||
output.print(&"=".repeat(60));
|
output.print(&"=".repeat(60));
|
||||||
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -1026,6 +1127,8 @@ async fn run_autonomous(
|
|||||||
" • Usage Percentage: {:.1}%",
|
" • Usage Percentage: {:.1}%",
|
||||||
context_window.percentage_used()
|
context_window.percentage_used()
|
||||||
));
|
));
|
||||||
|
// Add per-turn histogram
|
||||||
|
output.print(&generate_turn_histogram(&turn_metrics));
|
||||||
output.print(&"=".repeat(60));
|
output.print(&"=".repeat(60));
|
||||||
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -1055,6 +1158,8 @@ async fn run_autonomous(
|
|||||||
let mut implementation_approved = false;
|
let mut implementation_approved = false;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
|
let turn_start_time = Instant::now();
|
||||||
|
let turn_start_tokens = agent.get_context_window().used_tokens;
|
||||||
// Skip player turn if it's the first turn and implementation files exist
|
// Skip player turn if it's the first turn and implementation files exist
|
||||||
if !(turn == 1 && skip_first_player) {
|
if !(turn == 1 && skip_first_player) {
|
||||||
output.print(&format!(
|
output.print(&format!(
|
||||||
@@ -1150,6 +1255,8 @@ async fn run_autonomous(
|
|||||||
" • Usage Percentage: {:.1}%",
|
" • Usage Percentage: {:.1}%",
|
||||||
context_window.percentage_used()
|
context_window.percentage_used()
|
||||||
));
|
));
|
||||||
|
// Add per-turn histogram
|
||||||
|
output.print(&generate_turn_histogram(&turn_metrics));
|
||||||
output.print(&"=".repeat(60));
|
output.print(&"=".repeat(60));
|
||||||
|
|
||||||
return Err(e);
|
return Err(e);
|
||||||
@@ -1179,6 +1286,14 @@ async fn run_autonomous(
|
|||||||
"⚠️ Player turn {} failed after max retries. Moving to next turn.",
|
"⚠️ Player turn {} failed after max retries. Moving to next turn.",
|
||||||
turn
|
turn
|
||||||
));
|
));
|
||||||
|
// Record turn metrics before incrementing
|
||||||
|
let turn_duration = turn_start_time.elapsed();
|
||||||
|
let turn_tokens = agent.get_context_window().used_tokens - turn_start_tokens;
|
||||||
|
turn_metrics.push(TurnMetrics {
|
||||||
|
turn_number: turn,
|
||||||
|
tokens_used: turn_tokens,
|
||||||
|
wall_clock_time: turn_duration,
|
||||||
|
});
|
||||||
turn += 1;
|
turn += 1;
|
||||||
|
|
||||||
// Check if we've reached max turns
|
// Check if we've reached max turns
|
||||||
@@ -1292,6 +1407,8 @@ Remember: Be thorough in your review but concise in your feedback. APPROVE if th
|
|||||||
" • Usage Percentage: {:.1}%",
|
" • Usage Percentage: {:.1}%",
|
||||||
context_window.percentage_used()
|
context_window.percentage_used()
|
||||||
));
|
));
|
||||||
|
// Add per-turn histogram
|
||||||
|
output.print(&generate_turn_histogram(&turn_metrics));
|
||||||
output.print(&"=".repeat(60));
|
output.print(&"=".repeat(60));
|
||||||
|
|
||||||
return Err(e);
|
return Err(e);
|
||||||
@@ -1324,6 +1441,14 @@ Remember: Be thorough in your review but concise in your feedback. APPROVE if th
|
|||||||
turn
|
turn
|
||||||
));
|
));
|
||||||
coach_feedback = "The implementation needs review. Please ensure all requirements are met and the code compiles without errors.".to_string();
|
coach_feedback = "The implementation needs review. Please ensure all requirements are met and the code compiles without errors.".to_string();
|
||||||
|
// Record turn metrics before incrementing
|
||||||
|
let turn_duration = turn_start_time.elapsed();
|
||||||
|
let turn_tokens = agent.get_context_window().used_tokens - turn_start_tokens;
|
||||||
|
turn_metrics.push(TurnMetrics {
|
||||||
|
turn_number: turn,
|
||||||
|
tokens_used: turn_tokens,
|
||||||
|
wall_clock_time: turn_duration,
|
||||||
|
});
|
||||||
turn += 1;
|
turn += 1;
|
||||||
|
|
||||||
if turn > max_turns {
|
if turn > max_turns {
|
||||||
@@ -1351,6 +1476,14 @@ Remember: Be thorough in your review but concise in your feedback. APPROVE if th
|
|||||||
if coach_feedback_text.is_empty() {
|
if coach_feedback_text.is_empty() {
|
||||||
output.print("⚠️ Coach did not provide feedback. This may be a model issue.");
|
output.print("⚠️ Coach did not provide feedback. This may be a model issue.");
|
||||||
coach_feedback = "The implementation needs review. Please ensure all requirements are met and the code compiles without errors.".to_string();
|
coach_feedback = "The implementation needs review. Please ensure all requirements are met and the code compiles without errors.".to_string();
|
||||||
|
// Record turn metrics before incrementing
|
||||||
|
let turn_duration = turn_start_time.elapsed();
|
||||||
|
let turn_tokens = agent.get_context_window().used_tokens - turn_start_tokens;
|
||||||
|
turn_metrics.push(TurnMetrics {
|
||||||
|
turn_number: turn,
|
||||||
|
tokens_used: turn_tokens,
|
||||||
|
wall_clock_time: turn_duration,
|
||||||
|
});
|
||||||
turn += 1;
|
turn += 1;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -1374,6 +1507,14 @@ Remember: Be thorough in your review but concise in your feedback. APPROVE if th
|
|||||||
|
|
||||||
// Store coach feedback for next iteration
|
// Store coach feedback for next iteration
|
||||||
coach_feedback = coach_feedback_text;
|
coach_feedback = coach_feedback_text;
|
||||||
|
// Record turn metrics before incrementing
|
||||||
|
let turn_duration = turn_start_time.elapsed();
|
||||||
|
let turn_tokens = agent.get_context_window().used_tokens - turn_start_tokens;
|
||||||
|
turn_metrics.push(TurnMetrics {
|
||||||
|
turn_number: turn,
|
||||||
|
tokens_used: turn_tokens,
|
||||||
|
wall_clock_time: turn_duration,
|
||||||
|
});
|
||||||
turn += 1;
|
turn += 1;
|
||||||
|
|
||||||
output.print("🔄 Coach provided feedback for next iteration");
|
output.print("🔄 Coach provided feedback for next iteration");
|
||||||
@@ -1417,6 +1558,9 @@ Remember: Be thorough in your review but concise in your feedback. APPROVE if th
|
|||||||
" • Usage Percentage: {:.1}%",
|
" • Usage Percentage: {:.1}%",
|
||||||
context_window.percentage_used()
|
context_window.percentage_used()
|
||||||
));
|
));
|
||||||
|
|
||||||
|
// Add per-turn histogram
|
||||||
|
output.print(&generate_turn_histogram(&turn_metrics));
|
||||||
output.print(&"=".repeat(60));
|
output.print(&"=".repeat(60));
|
||||||
|
|
||||||
if implementation_approved {
|
if implementation_approved {
|
||||||
|
|||||||
Reference in New Issue
Block a user