Fix JSON filter to preserve code fence and indented content

Two cosmetic bugs fixed:
1. JSON inside code fences was being filtered - now tracks fence state
   and passes through all content inside ``` ... ``` blocks
2. Indented JSON was being filtered - now recognizes that real tool
   calls are never indented, so indented JSON is always documentation

Changes:
- Added in_code_fence and fence_buffer fields to FilterState
- Added track_code_fence() to detect ``` markers (with/without language)
- Added pass_through_char() for content inside code fences
- Modified '{' handling to only filter when no leading whitespace
- Added 4 new unit tests for code fence and indentation cases
- Updated 3 stress tests to expect new (correct) behavior

All 16 filter_json unit tests and 59 stress tests pass.
This commit is contained in:
Dhanji R. Prasanna
2026-01-19 17:00:43 +05:30
parent 1604ed613a
commit 6ff21a7d47
2 changed files with 109 additions and 4 deletions

View File

@@ -420,7 +420,8 @@ fn test_tabs_before_brace() {
reset_json_tool_state();
let input = "Text\n\t\t{\"tool\": \"x\", \"args\": {}}\nMore";
let result = filter_json_tool_calls(input);
assert_eq!(result, "Text\n\nMore");
// Indented JSON should NOT be filtered - real tool calls are never indented
assert_eq!(result, input);
}
#[test]
@@ -428,7 +429,8 @@ fn test_spaces_before_brace() {
reset_json_tool_state();
let input = "Text\n {\"tool\": \"x\", \"args\": {}}\nMore";
let result = filter_json_tool_calls(input);
assert_eq!(result, "Text\n\nMore");
// Indented JSON should NOT be filtered - real tool calls are never indented
assert_eq!(result, input);
}
#[test]
@@ -436,7 +438,8 @@ fn test_mixed_whitespace_before_brace() {
reset_json_tool_state();
let input = "Text\n \t \t {\"tool\": \"x\", \"args\": {}}\nMore";
let result = filter_json_tool_calls(input);
assert_eq!(result, "Text\n\nMore");
// Indented JSON should NOT be filtered - real tool calls are never indented
assert_eq!(result, input);
}
#[test]