Fix panic on multi-byte chars in filter_json buffer truncation
The buffer truncation code was slicing at a raw byte offset which could land in the middle of a multi-byte character (like emojis), causing a panic. Fixed by using char_indices() to find valid character boundaries. Also added stop_reason field to CompletionChunk initializers in tests to complete the stop_reason feature addition. - Fix byte boundary panic in filter_json.rs line 327 - Add test for multi-byte character handling - Update test files with missing stop_reason field
This commit is contained in:
@@ -13,6 +13,7 @@ fn chunk(content: &str, finished: bool) -> CompletionChunk {
|
||||
finished,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ fn test_has_incomplete_tool_call_no_tool_pattern() {
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
assert!(!parser.has_incomplete_tool_call());
|
||||
@@ -30,6 +31,7 @@ fn test_has_incomplete_tool_call_complete_tool_call() {
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
// Complete JSON should NOT be detected as incomplete
|
||||
@@ -45,6 +47,7 @@ fn test_has_incomplete_tool_call_truncated_tool_call() {
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
// Incomplete JSON should be detected
|
||||
@@ -60,6 +63,7 @@ fn test_has_incomplete_tool_call_truncated_mid_value() {
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
// Incomplete JSON should be detected
|
||||
@@ -77,6 +81,7 @@ fn test_has_incomplete_tool_call_with_text_before() {
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
// Incomplete JSON should be detected
|
||||
@@ -93,6 +98,7 @@ fn test_has_incomplete_tool_call_malformed_like_trace() {
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
// Truncated JSON (missing closing braces) should be detected as incomplete
|
||||
@@ -113,6 +119,7 @@ fn test_has_unexecuted_tool_call_no_tool_pattern() {
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
assert!(!parser.has_unexecuted_tool_call());
|
||||
@@ -126,6 +133,7 @@ fn test_has_unexecuted_tool_call_complete_tool_call() {
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
// Complete JSON tool call that wasn't executed should be detected
|
||||
@@ -140,6 +148,7 @@ fn test_has_unexecuted_tool_call_incomplete_json() {
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
// Incomplete JSON should NOT be detected as unexecuted (it's incomplete, not unexecuted)
|
||||
@@ -157,6 +166,7 @@ Some trailing text after the JSON"#.to_string(),
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
// Complete JSON tool call should be detected even with trailing text
|
||||
@@ -175,6 +185,7 @@ I'll execute this command now."#.to_string(),
|
||||
finished: false,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
};
|
||||
parser.process_chunk(&chunk);
|
||||
// Complete JSON tool call should be detected
|
||||
|
||||
@@ -17,6 +17,7 @@ fn chunk(content: &str, finished: bool) -> CompletionChunk {
|
||||
finished,
|
||||
tool_calls: None,
|
||||
usage: None,
|
||||
stop_reason: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user