more json filtering

This commit is contained in:
Dhanji Prasanna
2025-11-03 11:56:16 +11:00
parent 57d473c19d
commit 1f9fef5f18
2 changed files with 68 additions and 0 deletions

View File

@@ -106,6 +106,40 @@ pub fn fixed_filter_json_tool_calls(content: &str) -> String {
_ => {} _ => {}
} }
} }
// CRITICAL FIX: After counting braces, if still in suppression mode,
// check if a new tool call pattern appears. This handles truncated JSON
// followed by complete JSON.
if state.suppression_mode {
let current_json_start = state.json_start_in_buffer.unwrap();
// Don't require newline - the new JSON might be concatenated directly
let tool_call_regex = Regex::new(r#"\{\s*"tool"\s*:\s*""#).unwrap();
// Look for new tool call patterns after the current one
if let Some(captures) = tool_call_regex.find(&state.buffer[current_json_start + 1..]) {
let new_json_start = current_json_start + 1 + captures.start() + captures.as_str().find('{').unwrap();
debug!("Detected new tool call at position {} while processing incomplete one at {} - discarding old", new_json_start, current_json_start);
// The previous JSON was incomplete/malformed
// Return content before the old JSON (if any)
let content_before_old_json = if current_json_start > state.content_returned_up_to {
state.buffer[state.content_returned_up_to..current_json_start].to_string()
} else {
String::new()
};
// Update state to skip the incomplete JSON and position at the new one
// We'll process the new JSON on the next call
state.content_returned_up_to = new_json_start;
state.suppression_mode = false;
state.json_start_in_buffer = None;
state.brace_depth = 0;
return content_before_old_json;
}
}
// Still in suppression mode, return empty string (content is being accumulated) // Still in suppression mode, return empty string (content is being accumulated)
return String::new(); return String::new();
} }

View File

@@ -347,4 +347,38 @@ More text"#;
let expected = "Some text before\n\nText after"; let expected = "Some text before\n\nText after";
assert_eq!(final_result, expected); assert_eq!(final_result, expected);
} }
/// Test handling of truncated JSON followed by complete JSON (the json_err pattern)
#[test]
fn test_truncated_then_complete_json() {
reset_fixed_json_tool_state();
// Simulate the pattern from json_err trace:
// 1. Incomplete/truncated JSON appears
// 2. Then the same complete JSON appears
let chunks = vec![
"Some text\n",
r#"{"tool": "str_replace", "args": {"diff":"...","file_path":"./crates/g3-cli"#, // Truncated
r#"{"tool": "str_replace", "args": {"diff":"...","file_path":"./crates/g3-cli/src/lib.rs"}}"#, // Complete
"\nMore text",
];
let mut results = Vec::new();
for (i, chunk) in chunks.iter().enumerate() {
let result = fixed_filter_json_tool_calls(chunk);
println!("Chunk {}: {:?} -> {:?}", i, chunk, result);
results.push(result);
}
let final_result: String = results.join("");
println!("Final result: {:?}", final_result);
// The truncated JSON should be discarded when the complete one appears
// Both JSONs should be filtered out, leaving only the text
let expected = "Some text\n\nMore text";
assert_eq!(
final_result, expected,
"Failed to handle truncated JSON followed by complete JSON"
);
}
} }