Fix JSON filter to preserve code fence and indented content
Two cosmetic bugs fixed:
1. JSON inside code fences was being filtered - now tracks fence state
and passes through all content inside ``` ... ``` blocks
2. Indented JSON was being filtered - now recognizes that real tool
calls are never indented, so indented JSON is always documentation
Changes:
- Added in_code_fence and fence_buffer fields to FilterState
- Added track_code_fence() to detect ``` markers (with/without language)
- Added pass_through_char() for content inside code fences
- Modified '{' handling to only filter when no leading whitespace
- Added 4 new unit tests for code fence and indentation cases
- Updated 3 stress tests to expect new (correct) behavior
All 16 filter_json unit tests and 59 stress tests pass.
This commit is contained in:
@@ -420,7 +420,8 @@ fn test_tabs_before_brace() {
|
||||
reset_json_tool_state();
|
||||
let input = "Text\n\t\t{\"tool\": \"x\", \"args\": {}}\nMore";
|
||||
let result = filter_json_tool_calls(input);
|
||||
assert_eq!(result, "Text\n\nMore");
|
||||
// Indented JSON should NOT be filtered - real tool calls are never indented
|
||||
assert_eq!(result, input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -428,7 +429,8 @@ fn test_spaces_before_brace() {
|
||||
reset_json_tool_state();
|
||||
let input = "Text\n {\"tool\": \"x\", \"args\": {}}\nMore";
|
||||
let result = filter_json_tool_calls(input);
|
||||
assert_eq!(result, "Text\n\nMore");
|
||||
// Indented JSON should NOT be filtered - real tool calls are never indented
|
||||
assert_eq!(result, input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -436,7 +438,8 @@ fn test_mixed_whitespace_before_brace() {
|
||||
reset_json_tool_state();
|
||||
let input = "Text\n \t \t {\"tool\": \"x\", \"args\": {}}\nMore";
|
||||
let result = filter_json_tool_calls(input);
|
||||
assert_eq!(result, "Text\n\nMore");
|
||||
// Indented JSON should NOT be filtered - real tool calls are never indented
|
||||
assert_eq!(result, input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user