Readability improvements across streaming_parser, input_formatter, commands
- streaming_parser.rs: Reduced ~70 lines by removing redundant comments, consolidating doc comments, using slice syntax for TOOL_CALL_PATTERNS - input_formatter.rs: Lazy regex compilation via once_cell (performance), cleaner function structure, reduced comment noise - commands.rs: Extracted format_research_task_summary() and format_research_report_header() helpers, reduced ~40 lines of duplication - pending_research.rs: Fixed 2 unused variable warnings in tests All changes are behavior-preserving. 446 tests pass. Agent: carmack
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
//! Interactive command handlers for G3 CLI.
|
//! Interactive command handlers for G3 CLI.
|
||||||
//!
|
//!
|
||||||
//! Handles `/` commands in interactive mode.
|
//! Handles `/` commands in interactive mode (help, compact, research, etc.).
|
||||||
|
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use rustyline::Editor;
|
use rustyline::Editor;
|
||||||
@@ -16,6 +16,33 @@ use crate::project::load_and_validate_project;
|
|||||||
use crate::template::process_template;
|
use crate::template::process_template;
|
||||||
use crate::task_execution::execute_task_with_retry;
|
use crate::task_execution::execute_task_with_retry;
|
||||||
|
|
||||||
|
// --- Research command helpers ---
|
||||||
|
|
||||||
|
fn format_research_task_summary(task: &g3_core::pending_research::ResearchTask) -> String {
|
||||||
|
let status_emoji = match task.status {
|
||||||
|
g3_core::pending_research::ResearchStatus::Pending => "🔄",
|
||||||
|
g3_core::pending_research::ResearchStatus::Complete => "✅",
|
||||||
|
g3_core::pending_research::ResearchStatus::Failed => "❌",
|
||||||
|
};
|
||||||
|
let injected = if task.injected { " (injected)" } else { "" };
|
||||||
|
let query_preview = if task.query.len() > 60 {
|
||||||
|
format!("{}...", task.query.chars().take(57).collect::<String>())
|
||||||
|
} else {
|
||||||
|
task.query.clone()
|
||||||
|
};
|
||||||
|
format!(
|
||||||
|
" {} `{}` - {} ({}){}\n Query: {}",
|
||||||
|
status_emoji, task.id, task.status, task.elapsed_display(), injected, query_preview
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_research_report_header(task: &g3_core::pending_research::ResearchTask) -> String {
|
||||||
|
format!(
|
||||||
|
"📋 Research Report: `{}`\n\nQuery: {}\n\nStatus: {} | Elapsed: {}\n\n{}",
|
||||||
|
task.id, task.query, task.status, task.elapsed_display(), "─".repeat(60)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Handle a control command. Returns true if the command was handled and the loop should continue.
|
/// Handle a control command. Returns true if the command was handled and the loop should continue.
|
||||||
pub async fn handle_command<W: UiWriter>(
|
pub async fn handle_command<W: UiWriter>(
|
||||||
input: &str,
|
input: &str,
|
||||||
@@ -135,89 +162,49 @@ pub async fn handle_command<W: UiWriter>(
|
|||||||
}
|
}
|
||||||
cmd if cmd == "/research" || cmd.starts_with("/research ") => {
|
cmd if cmd == "/research" || cmd.starts_with("/research ") => {
|
||||||
let manager = agent.get_pending_research_manager();
|
let manager = agent.get_pending_research_manager();
|
||||||
|
|
||||||
// Parse argument: /research, /research latest, /research <id>
|
|
||||||
let arg = cmd.strip_prefix("/research").unwrap_or("").trim();
|
let arg = cmd.strip_prefix("/research").unwrap_or("").trim();
|
||||||
|
|
||||||
if arg.is_empty() {
|
if arg.is_empty() {
|
||||||
// List all research tasks
|
|
||||||
let all_tasks = manager.list_all();
|
let all_tasks = manager.list_all();
|
||||||
|
|
||||||
if all_tasks.is_empty() {
|
if all_tasks.is_empty() {
|
||||||
output.print("📋 No research tasks (pending or completed).");
|
output.print("📋 No research tasks (pending or completed).");
|
||||||
} else {
|
} else {
|
||||||
output.print(&format!("📋 Research Tasks ({} total):\n", all_tasks.len()));
|
output.print(&format!("📋 Research Tasks ({} total):\n", all_tasks.len()));
|
||||||
|
for task in all_tasks {
|
||||||
for task in all_tasks {
|
output.print(&format_research_task_summary(&task));
|
||||||
let status_emoji = match task.status {
|
output.print("");
|
||||||
g3_core::pending_research::ResearchStatus::Pending => "🔄",
|
|
||||||
g3_core::pending_research::ResearchStatus::Complete => "✅",
|
|
||||||
g3_core::pending_research::ResearchStatus::Failed => "❌",
|
|
||||||
};
|
|
||||||
|
|
||||||
let injected_marker = if task.injected { " (injected)" } else { "" };
|
|
||||||
|
|
||||||
output.print(&format!(
|
|
||||||
" {} `{}` - {} ({}){}\n Query: {}",
|
|
||||||
status_emoji,
|
|
||||||
task.id,
|
|
||||||
task.status,
|
|
||||||
task.elapsed_display(),
|
|
||||||
injected_marker,
|
|
||||||
if task.query.len() > 60 {
|
|
||||||
format!("{}...", &task.query.chars().take(57).collect::<String>())
|
|
||||||
} else {
|
|
||||||
task.query.clone()
|
|
||||||
}
|
|
||||||
));
|
|
||||||
output.print("");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if arg == "latest" {
|
} else if arg == "latest" {
|
||||||
// Show the most recent research report
|
|
||||||
let all_tasks = manager.list_all();
|
let all_tasks = manager.list_all();
|
||||||
|
|
||||||
// Find the most recent completed task (smallest elapsed time = most recent)
|
|
||||||
let latest = all_tasks.iter()
|
let latest = all_tasks.iter()
|
||||||
.filter(|t| t.status != g3_core::pending_research::ResearchStatus::Pending)
|
.filter(|t| t.status != g3_core::pending_research::ResearchStatus::Pending)
|
||||||
.min_by_key(|t| t.started_at.elapsed());
|
.min_by_key(|t| t.started_at.elapsed());
|
||||||
|
|
||||||
match latest {
|
match latest {
|
||||||
Some(task) => {
|
Some(task) => {
|
||||||
output.print(&format!("📋 Research Report: `{}`\n", task.id));
|
output.print(&format_research_report_header(task));
|
||||||
output.print(&format!("Query: {}\n", task.query));
|
output.print(task.result.as_deref().unwrap_or("(No report content available)"));
|
||||||
output.print(&format!("Status: {} | Elapsed: {}\n", task.status, task.elapsed_display()));
|
|
||||||
output.print(&"─".repeat(60));
|
|
||||||
if let Some(ref result) = task.result {
|
|
||||||
output.print(result);
|
|
||||||
} else {
|
|
||||||
output.print("(No report content available)");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
output.print("📋 No completed research tasks yet.");
|
output.print("📋 No completed research tasks yet.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// View a specific research report by ID
|
match manager.get(&arg.to_string()) {
|
||||||
let task_id = arg.to_string();
|
|
||||||
|
|
||||||
match manager.get(&task_id) {
|
|
||||||
Some(task) => {
|
Some(task) => {
|
||||||
output.print(&format!("📋 Research Report: `{}`\n", task.id));
|
output.print(&format_research_report_header(&task));
|
||||||
output.print(&format!("Query: {}\n", task.query));
|
let content = if let Some(ref result) = task.result {
|
||||||
output.print(&format!("Status: {} | Elapsed: {}\n", task.status, task.elapsed_display()));
|
result.as_str()
|
||||||
output.print(&"─".repeat(60));
|
|
||||||
if let Some(ref result) = task.result {
|
|
||||||
output.print(result);
|
|
||||||
} else if task.status == g3_core::pending_research::ResearchStatus::Pending {
|
} else if task.status == g3_core::pending_research::ResearchStatus::Pending {
|
||||||
output.print("(Research still in progress...)");
|
"(Research still in progress...)"
|
||||||
} else {
|
} else {
|
||||||
output.print("(No report content available)");
|
"(No report content available)"
|
||||||
}
|
};
|
||||||
|
output.print(content);
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
output.print(&format!("❓ No research task found with id: `{}`", task_id));
|
output.print(&format!("❓ No research task found with id: `{}`", arg));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,129 +1,102 @@
|
|||||||
//! Input formatting for interactive mode.
|
//! Input formatting for interactive mode.
|
||||||
//!
|
//!
|
||||||
//! Formats user input with markdown-style highlighting:
|
//! Applies visual highlighting to user input:
|
||||||
//! - ALL CAPS words become bold
|
//! - ALL CAPS words (2+ chars) → bold green
|
||||||
//! - Quoted text ("..." or '...') becomes cyan
|
//! - Quoted text ("..." or '...') → cyan
|
||||||
//! - Standard markdown formatting (bold, italic, code) is applied
|
//! - Standard markdown (bold, italic, code) via termimad
|
||||||
|
|
||||||
use crossterm::terminal;
|
use crossterm::terminal;
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::io::Write;
|
use std::io::Write;
|
||||||
use std::io::IsTerminal;
|
use std::io::IsTerminal;
|
||||||
|
use once_cell::sync::Lazy;
|
||||||
use termimad::MadSkin;
|
use termimad::MadSkin;
|
||||||
|
|
||||||
use crate::streaming_markdown::StreamingMarkdownFormatter;
|
use crate::streaming_markdown::StreamingMarkdownFormatter;
|
||||||
|
|
||||||
/// Pre-process input text to add markdown markers for special formatting.
|
// Compiled regexes for preprocessing (compiled once, reused)
|
||||||
///
|
static CAPS_RE: Lazy<Regex> = Lazy::new(|| {
|
||||||
/// This pass runs BEFORE markdown formatting:
|
// ALL CAPS words: 2+ uppercase letters, may include numbers, word boundaries
|
||||||
/// 1. ALL CAPS words (2+ chars) → wrapped in ** for bold
|
Regex::new(r"\b([A-Z][A-Z0-9]{1,}[A-Z0-9]*)\b").unwrap()
|
||||||
/// 2. Quoted text "..." or '...' → wrapped in special markers for cyan
|
});
|
||||||
///
|
static DOUBLE_QUOTE_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r#""([^"]+)""#).unwrap());
|
||||||
/// Returns the preprocessed text ready for markdown formatting.
|
static SINGLE_QUOTE_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"'([^']+)'").unwrap());
|
||||||
|
|
||||||
|
/// Pre-process input to add markdown markers before formatting.
|
||||||
|
/// ALL CAPS → **bold**, quoted text → special markers for cyan.
|
||||||
pub fn preprocess_input(input: &str) -> String {
|
pub fn preprocess_input(input: &str) -> String {
|
||||||
let mut result = input.to_string();
|
let mut result = input.to_string();
|
||||||
|
|
||||||
// First, handle ALL CAPS words (2+ uppercase letters, may include numbers)
|
// ALL CAPS → **bold**
|
||||||
// Must be a standalone word (word boundaries)
|
result = CAPS_RE.replace_all(&result, "**$1**").to_string();
|
||||||
let caps_re = Regex::new(r"\b([A-Z][A-Z0-9]{1,}[A-Z0-9]*)\b").unwrap();
|
|
||||||
result = caps_re.replace_all(&result, "**$1**").to_string();
|
|
||||||
|
|
||||||
// Then, handle quoted text - wrap in a special marker that we'll process after markdown
|
// Quoted text → markers (processed after markdown to apply cyan)
|
||||||
// Use lowercase placeholders that won't be matched by the ALL CAPS regex
|
result = DOUBLE_QUOTE_RE.replace_all(&result, "\x00qdbl\x00$1\x00qend\x00").to_string();
|
||||||
let double_quote_re = Regex::new(r#""([^"]+)""#).unwrap();
|
result = SINGLE_QUOTE_RE.replace_all(&result, "\x00qsgl\x00$1\x00qend\x00").to_string();
|
||||||
result = double_quote_re.replace_all(&result, "\x00qdbl\x00$1\x00qend\x00").to_string();
|
|
||||||
|
|
||||||
let single_quote_re = Regex::new(r"'([^']+)'").unwrap();
|
|
||||||
result = single_quote_re.replace_all(&result, "\x00qsgl\x00$1\x00qend\x00").to_string();
|
|
||||||
|
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Apply cyan highlighting to quoted text markers.
|
// Regexes for post-processing quote markers into ANSI cyan
|
||||||
/// This runs AFTER markdown formatting to apply the cyan color.
|
static CYAN_DOUBLE_RE: Lazy<Regex> = Lazy::new(|| {
|
||||||
|
Regex::new(r#"(\x1b\[36m")([^\x1b]*)\x1b\[0m"#).unwrap()
|
||||||
|
});
|
||||||
|
static CYAN_SINGLE_RE: Lazy<Regex> = Lazy::new(|| {
|
||||||
|
Regex::new(r"(\x1b\[36m')([^\x1b]*)\x1b\[0m").unwrap()
|
||||||
|
});
|
||||||
|
|
||||||
|
/// Apply cyan highlighting to quoted text markers (runs after markdown formatting).
|
||||||
fn apply_quote_highlighting(text: &str) -> String {
|
fn apply_quote_highlighting(text: &str) -> String {
|
||||||
let mut result = text.to_string();
|
let mut result = text.to_string();
|
||||||
|
|
||||||
// Replace double-quote markers with cyan formatting
|
|
||||||
// \x1b[36m = cyan, \x1b[0m = reset
|
// \x1b[36m = cyan, \x1b[0m = reset
|
||||||
result = result.replace("\x00qdbl\x00", "\x1b[36m\"");
|
result = result.replace("\x00qdbl\x00", "\x1b[36m\"");
|
||||||
result = result.replace("\x00qsgl\x00", "\x1b[36m'");
|
result = result.replace("\x00qsgl\x00", "\x1b[36m'");
|
||||||
result = result.replace("\x00qend\x00", "\x1b[0m");
|
result = result.replace("\x00qend\x00", "\x1b[0m");
|
||||||
|
|
||||||
// Add back the closing quotes
|
// Insert closing quotes before reset code
|
||||||
// We need to insert them before the reset code
|
result = CYAN_DOUBLE_RE.replace_all(&result, |caps: ®ex::Captures| {
|
||||||
let re = Regex::new(r#"(\x1b\[36m")([^\x1b]*)\x1b\[0m"#).unwrap();
|
|
||||||
result = re.replace_all(&result, |caps: ®ex::Captures| {
|
|
||||||
format!("{}{}\"\x1b[0m", &caps[1], &caps[2])
|
format!("{}{}\"\x1b[0m", &caps[1], &caps[2])
|
||||||
}).to_string();
|
}).to_string();
|
||||||
|
result = CYAN_SINGLE_RE.replace_all(&result, |caps: ®ex::Captures| {
|
||||||
let re = Regex::new(r"(\x1b\[36m')([^\x1b]*)\x1b\[0m").unwrap();
|
|
||||||
result = re.replace_all(&result, |caps: ®ex::Captures| {
|
|
||||||
format!("{}{}'\x1b[0m", &caps[1], &caps[2])
|
format!("{}{}'\x1b[0m", &caps[1], &caps[2])
|
||||||
}).to_string();
|
}).to_string();
|
||||||
|
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Format user input with markdown and special highlighting.
|
/// Format user input with markdown and special highlighting (ALL CAPS, quotes).
|
||||||
///
|
|
||||||
/// Applies:
|
|
||||||
/// 1. ALL CAPS → bold (green)
|
|
||||||
/// 2. Quoted text → cyan
|
|
||||||
/// 3. Standard markdown (bold, italic, inline code)
|
|
||||||
pub fn format_input(input: &str) -> String {
|
pub fn format_input(input: &str) -> String {
|
||||||
// Pre-process to add markdown markers
|
|
||||||
let preprocessed = preprocess_input(input);
|
let preprocessed = preprocess_input(input);
|
||||||
|
|
||||||
// Apply markdown formatting using the streaming formatter
|
|
||||||
let skin = MadSkin::default();
|
let skin = MadSkin::default();
|
||||||
let mut formatter = StreamingMarkdownFormatter::new(skin);
|
let mut formatter = StreamingMarkdownFormatter::new(skin);
|
||||||
let formatted = formatter.process(&preprocessed);
|
let formatted = formatter.process(&preprocessed);
|
||||||
let formatted = formatted + &formatter.finish();
|
let formatted = formatted + &formatter.finish();
|
||||||
|
|
||||||
// Apply quote highlighting (after markdown so colors don't interfere)
|
|
||||||
apply_quote_highlighting(&formatted)
|
apply_quote_highlighting(&formatted)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reprint user input in place with formatting.
|
/// Reprint user input in place with formatting (TTY only).
|
||||||
///
|
/// Moves cursor up to overwrite original input, then prints formatted version.
|
||||||
/// This moves the cursor up to overwrite the original input line,
|
|
||||||
/// then prints the formatted version.
|
|
||||||
///
|
|
||||||
/// Note: This function only performs formatting when stdout is a TTY.
|
|
||||||
/// In non-TTY contexts (piped output, etc.), it does nothing to avoid
|
|
||||||
/// corrupting terminal state for subsequent stdin operations.
|
|
||||||
pub fn reprint_formatted_input(input: &str, prompt: &str) {
|
pub fn reprint_formatted_input(input: &str, prompt: &str) {
|
||||||
// Only reformat if stdout is a TTY - avoid corrupting terminal state otherwise
|
|
||||||
if !std::io::stdout().is_terminal() {
|
if !std::io::stdout().is_terminal() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Format the input
|
|
||||||
let formatted = format_input(input);
|
let formatted = format_input(input);
|
||||||
|
|
||||||
// Get terminal width to calculate visual lines
|
// Calculate visual lines (prompt + input may wrap across terminal rows)
|
||||||
// The prompt + input may wrap across multiple terminal rows
|
let term_width = terminal::size().map(|(w, _)| w as usize).unwrap_or(80);
|
||||||
let term_width = terminal::size()
|
let visual_lines = (prompt.len() + input.len()).div_ceil(term_width).max(1);
|
||||||
.map(|(w, _)| w as usize)
|
|
||||||
.unwrap_or(80);
|
|
||||||
|
|
||||||
// Calculate visual lines: prompt + input length divided by terminal width
|
// Move up and clear each line
|
||||||
// This accounts for line wrapping in the terminal
|
|
||||||
let total_chars = prompt.len() + input.len();
|
|
||||||
let visual_lines = ((total_chars + term_width - 1) / term_width).max(1); // ceiling division
|
|
||||||
|
|
||||||
// Move cursor up by the number of lines and clear
|
|
||||||
for _ in 0..visual_lines {
|
for _ in 0..visual_lines {
|
||||||
// Move up one line and clear it
|
|
||||||
print!("\x1b[1A\x1b[2K");
|
print!("\x1b[1A\x1b[2K");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reprint with prompt and formatted input
|
// Dim prompt + formatted input
|
||||||
// Use dim color for the prompt to distinguish from the formatted input
|
|
||||||
println!("\x1b[2m{}\x1b[0m{}", prompt, formatted);
|
println!("\x1b[2m{}\x1b[0m{}", prompt, formatted);
|
||||||
|
|
||||||
// Ensure output is flushed
|
|
||||||
let _ = std::io::stdout().flush();
|
let _ = std::io::stdout().flush();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -366,7 +366,7 @@ mod tests {
|
|||||||
|
|
||||||
let id1 = manager.register("Query 1");
|
let id1 = manager.register("Query 1");
|
||||||
let id2 = manager.register("Query 2");
|
let id2 = manager.register("Query 2");
|
||||||
let id3 = manager.register("Query 3");
|
let _id3 = manager.register("Query 3");
|
||||||
|
|
||||||
// Complete two, leave one pending
|
// Complete two, leave one pending
|
||||||
manager.complete(&id1, "Report 1".to_string());
|
manager.complete(&id1, "Report 1".to_string());
|
||||||
@@ -415,7 +415,7 @@ mod tests {
|
|||||||
assert!(manager.format_status_summary().is_none());
|
assert!(manager.format_status_summary().is_none());
|
||||||
|
|
||||||
// One pending
|
// One pending
|
||||||
let id1 = manager.register("Query 1");
|
let _id1 = manager.register("Query 1");
|
||||||
let summary = manager.format_status_summary().unwrap();
|
let summary = manager.format_status_summary().unwrap();
|
||||||
assert!(summary.contains("1 researching"));
|
assert!(summary.contains("1 researching"));
|
||||||
|
|
||||||
|
|||||||
@@ -1,19 +1,21 @@
|
|||||||
//! Streaming tool parser for processing LLM response chunks.
|
//! Streaming tool parser for processing LLM response chunks.
|
||||||
//!
|
//!
|
||||||
//! This module handles parsing of tool calls from streaming LLM responses,
|
//! Parses tool calls from streaming LLM responses, supporting:
|
||||||
//! supporting both native tool calls and JSON-based fallback parsing.
|
//! - Native tool calls (returned directly by the provider)
|
||||||
|
//! - JSON-based fallback parsing (for embedded models)
|
||||||
//!
|
//!
|
||||||
//! **Important**: JSON tool calls are only recognized when they appear on their
|
//! # JSON Tool Call Recognition
|
||||||
//! own line (preceded by a newline or at the start of the buffer). This prevents
|
//!
|
||||||
//! inline JSON examples in prose from being incorrectly parsed as tool calls.
|
//! To prevent false positives from JSON examples in prose, tool calls are only
|
||||||
|
//! recognized when they appear "on their own line" - either at the start of the
|
||||||
|
//! buffer or preceded by a newline (with optional whitespace).
|
||||||
|
|
||||||
use tracing::debug;
|
use tracing::debug;
|
||||||
|
|
||||||
use crate::ToolCall;
|
use crate::ToolCall;
|
||||||
|
|
||||||
/// Patterns used to detect JSON tool calls in text.
|
/// JSON patterns that indicate a tool call. Covers common whitespace variations.
|
||||||
/// These cover common whitespace variations in JSON formatting.
|
const TOOL_CALL_PATTERNS: &[&str] = &[
|
||||||
const TOOL_CALL_PATTERNS: [&str; 4] = [
|
|
||||||
r#"{"tool":"#,
|
r#"{"tool":"#,
|
||||||
r#"{ "tool":"#,
|
r#"{ "tool":"#,
|
||||||
r#"{"tool" :"#,
|
r#"{"tool" :"#,
|
||||||
@@ -24,10 +26,7 @@ const TOOL_CALL_PATTERNS: [&str; 4] = [
|
|||||||
// Code Fence Tracking
|
// Code Fence Tracking
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
/// Tracks whether we're inside a markdown code fence (``` block).
|
/// Tracks code fence state to avoid parsing JSON examples inside ``` blocks.
|
||||||
///
|
|
||||||
/// Used during streaming to avoid parsing JSON examples inside code blocks
|
|
||||||
/// as tool calls.
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
struct CodeFenceTracker {
|
struct CodeFenceTracker {
|
||||||
/// Whether we're currently inside a code fence
|
/// Whether we're currently inside a code fence
|
||||||
@@ -53,10 +52,8 @@ impl CodeFenceTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if current_line is a code fence marker and toggle state if so.
|
|
||||||
fn check_and_toggle_fence(&mut self) {
|
fn check_and_toggle_fence(&mut self) {
|
||||||
let trimmed = self.current_line.trim_start();
|
if self.current_line.trim_start().starts_with("```") {
|
||||||
if trimmed.starts_with("```") && trimmed.chars().take_while(|&c| c == '`').count() >= 3 {
|
|
||||||
self.in_fence = !self.in_fence;
|
self.in_fence = !self.in_fence;
|
||||||
debug!(
|
debug!(
|
||||||
"Code fence toggled: in_fence={} (line: {:?})",
|
"Code fence toggled: in_fence={} (line: {:?})",
|
||||||
@@ -75,9 +72,7 @@ impl CodeFenceTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find all code fence ranges in text (for batch processing).
|
/// Find all code fence ranges in text. Returns (start, end) byte positions.
|
||||||
///
|
|
||||||
/// Returns a vector of (start, end) byte positions where code fences are.
|
|
||||||
/// Each range represents content INSIDE a fence (between ``` markers).
|
/// Each range represents content INSIDE a fence (between ``` markers).
|
||||||
fn find_code_fence_ranges(text: &str) -> Vec<(usize, usize)> {
|
fn find_code_fence_ranges(text: &str) -> Vec<(usize, usize)> {
|
||||||
let mut ranges = Vec::new();
|
let mut ranges = Vec::new();
|
||||||
@@ -121,8 +116,7 @@ fn is_position_in_fence_ranges(pos: usize, ranges: &[(usize, usize)]) -> bool {
|
|||||||
// JSON Parsing Utilities
|
// JSON Parsing Utilities
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
/// Find the end position (byte index) of a complete JSON object in the text.
|
/// Find the end byte index of a complete JSON object, or None if incomplete.
|
||||||
/// Returns None if no complete JSON object is found.
|
|
||||||
fn find_json_object_end(text: &str) -> Option<usize> {
|
fn find_json_object_end(text: &str) -> Option<usize> {
|
||||||
let mut brace_count = 0;
|
let mut brace_count = 0;
|
||||||
let mut in_string = false;
|
let mut in_string = false;
|
||||||
@@ -155,12 +149,12 @@ fn find_json_object_end(text: &str) -> Option<usize> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a partial JSON tool call has been invalidated by subsequent content.
|
/// Check if a partial JSON tool call has been invalidated.
|
||||||
///
|
///
|
||||||
/// Detects two invalidation cases:
|
/// Invalidation cases:
|
||||||
/// 1. Unescaped newline inside a JSON string (invalid JSON)
|
/// 1. Unescaped newline inside a JSON string (invalid JSON)
|
||||||
/// 2. Newline followed by non-JSON prose (e.g., regular text, not `"`, `{`, `}`, etc.)
|
/// 2. Newline followed by non-JSON prose (regular text)
|
||||||
/// 3. Newline followed by a new tool call pattern (`{"tool"`) - indicates abandoned fragment
|
/// 3. Newline followed by a new tool call pattern - indicates abandoned fragment
|
||||||
fn is_json_invalidated(json_text: &str) -> bool {
|
fn is_json_invalidated(json_text: &str) -> bool {
|
||||||
let mut in_string = false;
|
let mut in_string = false;
|
||||||
let mut escape_next = false;
|
let mut escape_next = false;
|
||||||
@@ -187,8 +181,7 @@ fn is_json_invalidated(json_text: &str) -> bool {
|
|||||||
|
|
||||||
// Check what comes after the newline
|
// Check what comes after the newline
|
||||||
if let Some(&(next_pos, next_ch)) = chars.peek() {
|
if let Some(&(next_pos, next_ch)) = chars.peek() {
|
||||||
// Check if this is the start of a NEW tool call pattern
|
// New tool call pattern = previous fragment was abandoned
|
||||||
// This indicates the previous JSON fragment was abandoned
|
|
||||||
let remaining = &json_text[next_pos..];
|
let remaining = &json_text[next_pos..];
|
||||||
if remaining.starts_with("{\"tool\"")
|
if remaining.starts_with("{\"tool\"")
|
||||||
|| remaining.starts_with("{ \"tool\"")
|
|| remaining.starts_with("{ \"tool\"")
|
||||||
@@ -198,7 +191,6 @@ fn is_json_invalidated(json_text: &str) -> bool {
|
|||||||
return true; // New tool call started, previous fragment is abandoned
|
return true; // New tool call started, previous fragment is abandoned
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if next char is valid JSON continuation
|
|
||||||
let valid_json_char = matches!(
|
let valid_json_char = matches!(
|
||||||
next_ch,
|
next_ch,
|
||||||
'"' | '{' | '}' | '[' | ']' | ':' | ',' | '-' | '0'..='9' | 't' | 'f' | 'n' | '\n'
|
'"' | '{' | '}' | '[' | ']' | ':' | ',' | '-' | '0'..='9' | 't' | 'f' | 'n' | '\n'
|
||||||
@@ -216,11 +208,8 @@ fn is_json_invalidated(json_text: &str) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Detect malformed tool calls where LLM prose leaked into JSON keys.
|
/// Detect malformed tool calls where LLM prose leaked into JSON keys.
|
||||||
///
|
|
||||||
/// When the LLM "stutters" or mixes formats, it sometimes emits JSON where
|
|
||||||
/// the keys are actually fragments of conversational text rather than valid
|
|
||||||
/// parameter names.
|
|
||||||
fn args_contain_prose_fragments(args: &serde_json::Map<String, serde_json::Value>) -> bool {
|
fn args_contain_prose_fragments(args: &serde_json::Map<String, serde_json::Value>) -> bool {
|
||||||
|
// When the LLM "stutters", keys may contain conversational text fragments
|
||||||
const PROSE_MARKERS: &[&str] = &[
|
const PROSE_MARKERS: &[&str] = &[
|
||||||
"I'll", "Let me", "Here's", "I can", "I need", "First", "Now", "The ",
|
"I'll", "Let me", "Here's", "I can", "I need", "First", "Now", "The ",
|
||||||
];
|
];
|
||||||
@@ -236,9 +225,7 @@ fn args_contain_prose_fragments(args: &serde_json::Map<String, serde_json::Value
|
|||||||
// Tool Call Pattern Matching
|
// Tool Call Pattern Matching
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
/// Check if a position in text is "on its own line" - meaning it's either
|
/// True if position is at start of text or preceded only by whitespace after newline.
|
||||||
/// at the start of the text, or preceded by a newline with only whitespace
|
|
||||||
/// between the newline and the position.
|
|
||||||
fn is_on_own_line(text: &str, pos: usize) -> bool {
|
fn is_on_own_line(text: &str, pos: usize) -> bool {
|
||||||
if pos == 0 {
|
if pos == 0 {
|
||||||
return true;
|
return true;
|
||||||
@@ -247,22 +234,19 @@ fn is_on_own_line(text: &str, pos: usize) -> bool {
|
|||||||
text[line_start..pos].chars().all(|c| c.is_whitespace())
|
text[line_start..pos].chars().all(|c| c.is_whitespace())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find the first tool call pattern that appears on its own line.
|
|
||||||
fn find_first_tool_call_start(text: &str) -> Option<usize> {
|
fn find_first_tool_call_start(text: &str) -> Option<usize> {
|
||||||
find_tool_call_start(text, false)
|
find_tool_call_start(text, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find the last tool call pattern that appears on its own line.
|
|
||||||
fn find_last_tool_call_start(text: &str) -> Option<usize> {
|
fn find_last_tool_call_start(text: &str) -> Option<usize> {
|
||||||
find_tool_call_start(text, true)
|
find_tool_call_start(text, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find a tool call pattern in text, optionally searching backwards.
|
/// Find a tool call pattern on its own line. If `find_last`, search backwards.
|
||||||
/// Only matches patterns on their own line (at start or after newline + whitespace).
|
|
||||||
fn find_tool_call_start(text: &str, find_last: bool) -> Option<usize> {
|
fn find_tool_call_start(text: &str, find_last: bool) -> Option<usize> {
|
||||||
let mut best_pos: Option<usize> = None;
|
let mut best_pos: Option<usize> = None;
|
||||||
|
|
||||||
for pattern in &TOOL_CALL_PATTERNS {
|
for pattern in TOOL_CALL_PATTERNS {
|
||||||
if find_last {
|
if find_last {
|
||||||
// Search backwards
|
// Search backwards
|
||||||
let mut search_end = text.len();
|
let mut search_end = text.len();
|
||||||
@@ -306,20 +290,16 @@ fn find_tool_call_start(text: &str, find_last: bool) -> Option<usize> {
|
|||||||
// StreamingToolParser
|
// StreamingToolParser
|
||||||
// ============================================================================
|
// ============================================================================
|
||||||
|
|
||||||
/// Modern streaming tool parser that properly handles native tool calls and SSE chunks.
|
/// Streaming parser for tool calls from LLM responses (native or JSON fallback).
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct StreamingToolParser {
|
pub struct StreamingToolParser {
|
||||||
/// Buffer for accumulating text content
|
|
||||||
text_buffer: String,
|
text_buffer: String,
|
||||||
/// Position in text_buffer up to which tool calls have been consumed/executed.
|
|
||||||
last_consumed_position: usize,
|
last_consumed_position: usize,
|
||||||
/// Whether we've received a message_stop event
|
|
||||||
message_stopped: bool,
|
message_stopped: bool,
|
||||||
/// Whether we're currently in a JSON tool call (for fallback parsing)
|
// JSON fallback parsing state
|
||||||
in_json_tool_call: bool,
|
in_json_tool_call: bool,
|
||||||
/// Start position of JSON tool call (for fallback parsing)
|
|
||||||
json_tool_start: Option<usize>,
|
json_tool_start: Option<usize>,
|
||||||
/// Tracks code fence state during streaming
|
// Code fence tracking (to skip JSON examples in ``` blocks)
|
||||||
fence_tracker: CodeFenceTracker,
|
fence_tracker: CodeFenceTracker,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -345,13 +325,11 @@ impl StreamingToolParser {
|
|||||||
pub fn process_chunk(&mut self, chunk: &g3_providers::CompletionChunk) -> Vec<ToolCall> {
|
pub fn process_chunk(&mut self, chunk: &g3_providers::CompletionChunk) -> Vec<ToolCall> {
|
||||||
let mut completed_tools = Vec::new();
|
let mut completed_tools = Vec::new();
|
||||||
|
|
||||||
// Add text content to buffer and track code fence state
|
|
||||||
if !chunk.content.is_empty() {
|
if !chunk.content.is_empty() {
|
||||||
self.fence_tracker.process(&chunk.content);
|
self.fence_tracker.process(&chunk.content);
|
||||||
self.text_buffer.push_str(&chunk.content);
|
self.text_buffer.push_str(&chunk.content);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle native tool calls - return them immediately when received
|
|
||||||
if let Some(ref tool_calls) = chunk.tool_calls {
|
if let Some(ref tool_calls) = chunk.tool_calls {
|
||||||
debug!("Received native tool calls: {:?}", tool_calls);
|
debug!("Received native tool calls: {:?}", tool_calls);
|
||||||
for tool_call in tool_calls {
|
for tool_call in tool_calls {
|
||||||
@@ -362,10 +340,8 @@ impl StreamingToolParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if message is finished/stopped
|
|
||||||
if chunk.finished {
|
if chunk.finished {
|
||||||
self.message_stopped = true;
|
self.message_stopped = true;
|
||||||
debug!("Message finished, processing accumulated tool calls");
|
|
||||||
|
|
||||||
// When stream finishes, find ALL JSON tool calls in the accumulated buffer
|
// When stream finishes, find ALL JSON tool calls in the accumulated buffer
|
||||||
if completed_tools.is_empty() && !self.text_buffer.is_empty() {
|
if completed_tools.is_empty() && !self.text_buffer.is_empty() {
|
||||||
@@ -380,8 +356,7 @@ impl StreamingToolParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: Try to parse JSON tool calls from current chunk content if no native tool calls.
|
// JSON fallback: try to parse if no native calls and not inside a code fence
|
||||||
// Skip when inside a code fence to prevent false positives from JSON examples.
|
|
||||||
if completed_tools.is_empty()
|
if completed_tools.is_empty()
|
||||||
&& !chunk.content.is_empty()
|
&& !chunk.content.is_empty()
|
||||||
&& !chunk.finished
|
&& !chunk.finished
|
||||||
@@ -395,14 +370,11 @@ impl StreamingToolParser {
|
|||||||
completed_tools
|
completed_tools
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Try to parse a JSON tool call from the streaming buffer.
|
/// Try to parse a JSON tool call, tracking partial state across chunks.
|
||||||
///
|
|
||||||
/// Maintains state (`in_json_tool_call`, `json_tool_start`) to track
|
|
||||||
/// partial JSON tool calls across streaming chunks.
|
|
||||||
fn try_parse_streaming_json_tool_call(&mut self) -> Option<ToolCall> {
|
fn try_parse_streaming_json_tool_call(&mut self) -> Option<ToolCall> {
|
||||||
let fence_ranges = find_code_fence_ranges(&self.text_buffer);
|
let fence_ranges = find_code_fence_ranges(&self.text_buffer);
|
||||||
|
|
||||||
// If not currently in a JSON tool call, look for the start
|
// Look for the start of a new tool call
|
||||||
if !self.in_json_tool_call {
|
if !self.in_json_tool_call {
|
||||||
let unchecked_buffer = &self.text_buffer[self.last_consumed_position..];
|
let unchecked_buffer = &self.text_buffer[self.last_consumed_position..];
|
||||||
if let Some(relative_pos) = find_first_tool_call_start(unchecked_buffer) {
|
if let Some(relative_pos) = find_first_tool_call_start(unchecked_buffer) {
|
||||||
@@ -428,7 +400,6 @@ impl StreamingToolParser {
|
|||||||
if let Some(start_pos) = self.json_tool_start {
|
if let Some(start_pos) = self.json_tool_start {
|
||||||
let json_text = &self.text_buffer[start_pos..];
|
let json_text = &self.text_buffer[start_pos..];
|
||||||
|
|
||||||
// Try to find a complete JSON object
|
|
||||||
if let Some(end_pos) = find_json_object_end(json_text) {
|
if let Some(end_pos) = find_json_object_end(json_text) {
|
||||||
let json_str = &json_text[..=end_pos];
|
let json_str = &json_text[..=end_pos];
|
||||||
debug!("Attempting to parse JSON tool call: {}", json_str);
|
debug!("Attempting to parse JSON tool call: {}", json_str);
|
||||||
@@ -439,12 +410,10 @@ impl StreamingToolParser {
|
|||||||
return Some(tool_call);
|
return Some(tool_call);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse failed, reset and continue looking
|
|
||||||
self.in_json_tool_call = false;
|
self.in_json_tool_call = false;
|
||||||
self.json_tool_start = None;
|
self.json_tool_start = None;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if the partial JSON has been invalidated
|
|
||||||
if self.in_json_tool_call && is_json_invalidated(json_text) {
|
if self.in_json_tool_call && is_json_invalidated(json_text) {
|
||||||
debug!("JSON tool call invalidated by subsequent content, clearing state");
|
debug!("JSON tool call invalidated by subsequent content, clearing state");
|
||||||
self.in_json_tool_call = false;
|
self.in_json_tool_call = false;
|
||||||
@@ -458,7 +427,7 @@ impl StreamingToolParser {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse ALL JSON tool calls from the accumulated text buffer.
|
/// Parse all JSON tool calls from the accumulated buffer (used at stream end).
|
||||||
fn parse_all_json_tool_calls(&self) -> Vec<ToolCall> {
|
fn parse_all_json_tool_calls(&self) -> Vec<ToolCall> {
|
||||||
let mut tool_calls = Vec::new();
|
let mut tool_calls = Vec::new();
|
||||||
let mut search_start = 0;
|
let mut search_start = 0;
|
||||||
@@ -472,7 +441,6 @@ impl StreamingToolParser {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let abs_start = search_start + relative_pos;
|
let abs_start = search_start + relative_pos;
|
||||||
let json_text = &self.text_buffer[abs_start..];
|
|
||||||
|
|
||||||
// Skip if inside a code fence
|
// Skip if inside a code fence
|
||||||
if is_position_in_fence_ranges(abs_start, &fence_ranges) {
|
if is_position_in_fence_ranges(abs_start, &fence_ranges) {
|
||||||
@@ -480,7 +448,7 @@ impl StreamingToolParser {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to find a complete JSON object
|
let json_text = &self.text_buffer[abs_start..];
|
||||||
let Some(end_pos) = find_json_object_end(json_text) else {
|
let Some(end_pos) = find_json_object_end(json_text) else {
|
||||||
break; // Incomplete JSON, stop searching
|
break; // Incomplete JSON, stop searching
|
||||||
};
|
};
|
||||||
@@ -497,31 +465,22 @@ impl StreamingToolParser {
|
|||||||
tool_calls
|
tool_calls
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Try to parse a JSON string as a ToolCall, validating the args.
|
|
||||||
fn try_parse_tool_call_json(&self, json_str: &str) -> Option<ToolCall> {
|
fn try_parse_tool_call_json(&self, json_str: &str) -> Option<ToolCall> {
|
||||||
let tool_call: ToolCall = serde_json::from_str(json_str).ok()?;
|
let tool_call: ToolCall = serde_json::from_str(json_str).ok()?;
|
||||||
|
|
||||||
// Validate that args is an object with reasonable keys
|
|
||||||
let args_obj = tool_call.args.as_object()?;
|
let args_obj = tool_call.args.as_object()?;
|
||||||
|
|
||||||
if args_contain_prose_fragments(args_obj) {
|
if args_contain_prose_fragments(args_obj) {
|
||||||
debug!("Detected malformed tool call with message-like keys, skipping");
|
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
debug!("Successfully parsed valid JSON tool call: {:?}", tool_call);
|
|
||||||
Some(tool_call)
|
Some(tool_call)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ========================================================================
|
// --- Public Accessors ---
|
||||||
// Public Accessors
|
|
||||||
// ========================================================================
|
|
||||||
|
|
||||||
/// Get the accumulated text content.
|
|
||||||
pub fn get_text_content(&self) -> &str {
|
pub fn get_text_content(&self) -> &str {
|
||||||
&self.text_buffer
|
&self.text_buffer
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get content before a specific position (for display purposes).
|
|
||||||
pub fn get_content_before_position(&self, pos: usize) -> String {
|
pub fn get_content_before_position(&self, pos: usize) -> String {
|
||||||
if pos <= self.text_buffer.len() {
|
if pos <= self.text_buffer.len() {
|
||||||
self.text_buffer[..pos].to_string()
|
self.text_buffer[..pos].to_string()
|
||||||
@@ -530,12 +489,10 @@ impl StreamingToolParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if the message has been stopped/finished.
|
|
||||||
pub fn is_message_stopped(&self) -> bool {
|
pub fn is_message_stopped(&self) -> bool {
|
||||||
self.message_stopped
|
self.message_stopped
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if the text buffer contains an incomplete JSON tool call.
|
|
||||||
pub fn has_incomplete_tool_call(&self) -> bool {
|
pub fn has_incomplete_tool_call(&self) -> bool {
|
||||||
let unchecked_buffer = &self.text_buffer[self.last_consumed_position..];
|
let unchecked_buffer = &self.text_buffer[self.last_consumed_position..];
|
||||||
let Some(start_pos) = find_last_tool_call_start(unchecked_buffer) else {
|
let Some(start_pos) = find_last_tool_call_start(unchecked_buffer) else {
|
||||||
@@ -544,7 +501,6 @@ impl StreamingToolParser {
|
|||||||
|
|
||||||
let json_text = &unchecked_buffer[start_pos..];
|
let json_text = &unchecked_buffer[start_pos..];
|
||||||
|
|
||||||
// Complete or invalidated = not incomplete
|
|
||||||
if find_json_object_end(json_text).is_some() || is_json_invalidated(json_text) {
|
if find_json_object_end(json_text).is_some() || is_json_invalidated(json_text) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -552,7 +508,6 @@ impl StreamingToolParser {
|
|||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if the text buffer contains an unexecuted tool call.
|
|
||||||
pub fn has_unexecuted_tool_call(&self) -> bool {
|
pub fn has_unexecuted_tool_call(&self) -> bool {
|
||||||
let unchecked_buffer = &self.text_buffer[self.last_consumed_position..];
|
let unchecked_buffer = &self.text_buffer[self.last_consumed_position..];
|
||||||
let Some(start_pos) = find_last_tool_call_start(unchecked_buffer) else {
|
let Some(start_pos) = find_last_tool_call_start(unchecked_buffer) else {
|
||||||
@@ -568,27 +523,22 @@ impl StreamingToolParser {
|
|||||||
serde_json::from_str::<serde_json::Value>(json_only).is_ok()
|
serde_json::from_str::<serde_json::Value>(json_only).is_ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Mark all tool calls up to the current buffer position as consumed/executed.
|
|
||||||
pub fn mark_tool_calls_consumed(&mut self) {
|
pub fn mark_tool_calls_consumed(&mut self) {
|
||||||
self.last_consumed_position = self.text_buffer.len();
|
self.last_consumed_position = self.text_buffer.len();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the current text buffer length (for position tracking).
|
|
||||||
pub fn text_buffer_len(&self) -> usize {
|
pub fn text_buffer_len(&self) -> usize {
|
||||||
self.text_buffer.len()
|
self.text_buffer.len()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if currently parsing a JSON tool call (for debugging).
|
|
||||||
pub fn is_in_json_tool_call(&self) -> bool {
|
pub fn is_in_json_tool_call(&self) -> bool {
|
||||||
self.in_json_tool_call
|
self.in_json_tool_call
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the JSON tool start position (for debugging).
|
|
||||||
pub fn json_tool_start_position(&self) -> Option<usize> {
|
pub fn json_tool_start_position(&self) -> Option<usize> {
|
||||||
self.json_tool_start
|
self.json_tool_start
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reset the parser state for a new message.
|
|
||||||
pub fn reset(&mut self) {
|
pub fn reset(&mut self) {
|
||||||
self.text_buffer.clear();
|
self.text_buffer.clear();
|
||||||
self.last_consumed_position = 0;
|
self.last_consumed_position = 0;
|
||||||
@@ -598,34 +548,25 @@ impl StreamingToolParser {
|
|||||||
self.fence_tracker.reset();
|
self.fence_tracker.reset();
|
||||||
}
|
}
|
||||||
|
|
||||||
// ========================================================================
|
// --- Static Methods (for external use) ---
|
||||||
// Static Methods (for external use)
|
|
||||||
// ========================================================================
|
|
||||||
|
|
||||||
/// Find the starting position of the FIRST tool call pattern on its own line.
|
|
||||||
pub fn find_first_tool_call_start(text: &str) -> Option<usize> {
|
pub fn find_first_tool_call_start(text: &str) -> Option<usize> {
|
||||||
find_first_tool_call_start(text)
|
find_first_tool_call_start(text)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find the starting position of the LAST tool call pattern on its own line.
|
|
||||||
pub fn find_last_tool_call_start(text: &str) -> Option<usize> {
|
pub fn find_last_tool_call_start(text: &str) -> Option<usize> {
|
||||||
find_last_tool_call_start(text)
|
find_last_tool_call_start(text)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a position in text is "on its own line".
|
|
||||||
pub fn is_on_own_line(text: &str, pos: usize) -> bool {
|
pub fn is_on_own_line(text: &str, pos: usize) -> bool {
|
||||||
is_on_own_line(text, pos)
|
is_on_own_line(text, pos)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find the end position of a complete JSON object.
|
|
||||||
pub fn find_complete_json_object_end(text: &str) -> Option<usize> {
|
pub fn find_complete_json_object_end(text: &str) -> Option<usize> {
|
||||||
find_json_object_end(text)
|
find_json_object_end(text)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ============================================================================
|
|
||||||
// Tests
|
// Tests
|
||||||
// ============================================================================
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
|||||||
Reference in New Issue
Block a user