Auto-resize large images (>=5MB) in read_image tool
Images >= 5MB are now automatically resized to < 4.9MB using ImageMagick before being sent to the LLM. This prevents API errors from oversized images. - Uses iterative quality/scale reduction to find optimal size - Converts to JPEG for better compression - Shows original and resized size in terminal output (e.g., '6.2 MB → 4.1 MB (resized)') - Falls back to original if ImageMagick fails or isn't available
This commit is contained in:
@@ -13,6 +13,9 @@ use crate::ToolCall;
|
||||
|
||||
use super::executor::ToolContext;
|
||||
|
||||
/// Maximum image size in bytes (5MB) - images larger than this will be resized
|
||||
const MAX_IMAGE_SIZE: usize = 5 * 1024 * 1024;
|
||||
|
||||
/// Bytes per token heuristic (conservative estimate for code/text mix)
|
||||
const BYTES_PER_TOKEN: f32 = 3.5;
|
||||
|
||||
@@ -313,6 +316,28 @@ pub async fn execute_read_image<W: UiWriter>(
|
||||
}
|
||||
};
|
||||
|
||||
let original_size = bytes.len();
|
||||
|
||||
// Resize image if it's >= 5MB (target < 4.9MB to leave margin)
|
||||
let (bytes, was_resized) = if original_size >= MAX_IMAGE_SIZE {
|
||||
match resize_image_if_needed(&bytes, path, MAX_IMAGE_SIZE - 100 * 1024) {
|
||||
Ok(resized) => {
|
||||
let resized_size = resized.len();
|
||||
if resized_size < original_size {
|
||||
(resized, true)
|
||||
} else {
|
||||
(bytes, false)
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Failed to resize image: {}", e);
|
||||
(bytes, false)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(bytes, false)
|
||||
};
|
||||
|
||||
let file_size = bytes.len();
|
||||
|
||||
// Try to get image dimensions
|
||||
@@ -323,7 +348,23 @@ pub async fn execute_read_image<W: UiWriter>(
|
||||
.map(|(w, h)| format!("{}x{}", w, h))
|
||||
.unwrap_or_else(|| "unknown".to_string());
|
||||
|
||||
let size_str = if file_size >= 1024 * 1024 {
|
||||
let format_size = |size: usize| -> String {
|
||||
if size >= 1024 * 1024 {
|
||||
format!("{:.1} MB", size as f64 / (1024.0 * 1024.0))
|
||||
} else if size >= 1024 {
|
||||
format!("{:.1} KB", size as f64 / 1024.0)
|
||||
} else {
|
||||
format!("{} bytes", size)
|
||||
}
|
||||
};
|
||||
|
||||
let size_str = if was_resized {
|
||||
format!(
|
||||
"{} → {} (resized)",
|
||||
format_size(original_size),
|
||||
format_size(file_size)
|
||||
)
|
||||
} else if file_size >= 1024 * 1024 {
|
||||
format!("{:.1} MB", file_size as f64 / (1024.0 * 1024.0))
|
||||
} else if file_size >= 1024 {
|
||||
format!("{:.1} KB", file_size as f64 / 1024.0)
|
||||
@@ -331,13 +372,16 @@ pub async fn execute_read_image<W: UiWriter>(
|
||||
format!("{} bytes", file_size)
|
||||
};
|
||||
|
||||
// If resized, the output is JPEG
|
||||
let final_media_type = if was_resized { "image/jpeg" } else { media_type };
|
||||
|
||||
// Output imgcat inline image to terminal (height constrained)
|
||||
print_imgcat(&bytes, path_str, &dim_str, media_type, &size_str, 5);
|
||||
print_imgcat(&bytes, path_str, &dim_str, final_media_type, &size_str, 5);
|
||||
|
||||
// Store the image to be attached to the next user message
|
||||
use base64::Engine;
|
||||
let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes);
|
||||
let image = g3_providers::ImageContent::new(media_type, encoded);
|
||||
let image = g3_providers::ImageContent::new(final_media_type, encoded);
|
||||
ctx.pending_images.push(image);
|
||||
|
||||
success_count += 1;
|
||||
@@ -533,6 +577,81 @@ fn extract_path_and_content(args: &serde_json::Value) -> (Option<&str>, Option<&
|
||||
}
|
||||
}
|
||||
|
||||
/// Resize an image to be under the target size using ImageMagick.
|
||||
/// Returns the resized image bytes, or the original bytes if resizing fails or isn't needed.
|
||||
///
|
||||
/// Uses iterative quality reduction and dimension scaling to achieve target size.
|
||||
pub fn resize_image_if_needed(
|
||||
bytes: &[u8],
|
||||
path: &std::path::Path,
|
||||
target_size: usize,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
// If already under target size, return original
|
||||
if bytes.len() < target_size {
|
||||
return Ok(bytes.to_vec());
|
||||
}
|
||||
|
||||
debug!(
|
||||
"Image {} is {} bytes, resizing to under {} bytes",
|
||||
path.display(),
|
||||
bytes.len(),
|
||||
target_size
|
||||
);
|
||||
|
||||
// Create a temp file for the input
|
||||
let temp_dir = std::env::temp_dir();
|
||||
let input_path = temp_dir.join(format!("g3_resize_input_{}", std::process::id()));
|
||||
let output_path = temp_dir.join(format!("g3_resize_output_{}.jpg", std::process::id()));
|
||||
|
||||
// Write input bytes to temp file
|
||||
std::fs::write(&input_path, bytes)?;
|
||||
|
||||
// Try different quality levels, starting high and decreasing
|
||||
let quality_levels = [85, 70, 55, 40, 25];
|
||||
let scale_factors = [100, 80, 60, 50, 40];
|
||||
|
||||
for &scale in &scale_factors {
|
||||
for &quality in &quality_levels {
|
||||
// Use ImageMagick convert to resize
|
||||
let result = std::process::Command::new("convert")
|
||||
.arg(&input_path)
|
||||
.arg("-resize")
|
||||
.arg(format!("{}%", scale))
|
||||
.arg("-quality")
|
||||
.arg(format!("{}", quality))
|
||||
.arg(&output_path)
|
||||
.output();
|
||||
|
||||
if let Ok(output) = result {
|
||||
if output.status.success() {
|
||||
if let Ok(resized_bytes) = std::fs::read(&output_path) {
|
||||
if resized_bytes.len() < target_size {
|
||||
debug!(
|
||||
"Resized image to {} bytes (scale={}%, quality={})",
|
||||
resized_bytes.len(),
|
||||
scale,
|
||||
quality
|
||||
);
|
||||
// Clean up temp files
|
||||
let _ = std::fs::remove_file(&input_path);
|
||||
let _ = std::fs::remove_file(&output_path);
|
||||
return Ok(resized_bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up temp files
|
||||
let _ = std::fs::remove_file(&input_path);
|
||||
let _ = std::fs::remove_file(&output_path);
|
||||
|
||||
// If all attempts failed, return original bytes
|
||||
debug!("Failed to resize image under target size, using original");
|
||||
Ok(bytes.to_vec())
|
||||
}
|
||||
|
||||
/// Get image dimensions from raw bytes.
|
||||
pub fn get_image_dimensions(bytes: &[u8], media_type: &str) -> Option<(u32, u32)> {
|
||||
match media_type {
|
||||
|
||||
@@ -199,3 +199,32 @@ fn test_image_dimensions_gif() {
|
||||
assert_eq!(width, 100);
|
||||
assert_eq!(height, 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resize_image_if_needed_small_image() {
|
||||
use g3_core::tools::file_ops::resize_image_if_needed;
|
||||
use std::path::Path;
|
||||
|
||||
// Small image should not be resized
|
||||
let small_bytes = vec![0u8; 1000]; // 1KB
|
||||
let path = Path::new("test.jpg");
|
||||
let target_size = 5 * 1024 * 1024; // 5MB
|
||||
|
||||
let result = resize_image_if_needed(&small_bytes, path, target_size).unwrap();
|
||||
assert_eq!(result.len(), small_bytes.len(), "Small image should not be resized");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resize_image_if_needed_returns_original_on_failure() {
|
||||
use g3_core::tools::file_ops::resize_image_if_needed;
|
||||
use std::path::Path;
|
||||
|
||||
// Invalid image data - ImageMagick will fail, should return original
|
||||
let invalid_bytes = vec![0u8; 6 * 1024 * 1024]; // 6MB of zeros
|
||||
let path = Path::new("test.jpg");
|
||||
let target_size = 5 * 1024 * 1024; // 5MB
|
||||
|
||||
let result = resize_image_if_needed(&invalid_bytes, path, target_size).unwrap();
|
||||
// Should return original since ImageMagick can't process invalid data
|
||||
assert_eq!(result.len(), invalid_bytes.len(), "Invalid image should return original");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user