Auto-resize large images (>=5MB) in read_image tool

Images >= 5MB are now automatically resized to < 4.9MB using ImageMagick
before being sent to the LLM. This prevents API errors from oversized images.

- Uses iterative quality/scale reduction to find optimal size
- Converts to JPEG for better compression
- Shows original and resized size in terminal output (e.g., '6.2 MB → 4.1 MB (resized)')
- Falls back to original if ImageMagick fails or isn't available
This commit is contained in:
Dhanji R. Prasanna
2026-01-16 21:09:38 +05:30
parent fc702168ab
commit 1003386f7f
2 changed files with 151 additions and 3 deletions

View File

@@ -13,6 +13,9 @@ use crate::ToolCall;
use super::executor::ToolContext;
/// Maximum image size in bytes (5MB) - images larger than this will be resized
const MAX_IMAGE_SIZE: usize = 5 * 1024 * 1024;
/// Bytes per token heuristic (conservative estimate for code/text mix)
const BYTES_PER_TOKEN: f32 = 3.5;
@@ -313,6 +316,28 @@ pub async fn execute_read_image<W: UiWriter>(
}
};
let original_size = bytes.len();
// Resize image if it's >= 5MB (target < 4.9MB to leave margin)
let (bytes, was_resized) = if original_size >= MAX_IMAGE_SIZE {
match resize_image_if_needed(&bytes, path, MAX_IMAGE_SIZE - 100 * 1024) {
Ok(resized) => {
let resized_size = resized.len();
if resized_size < original_size {
(resized, true)
} else {
(bytes, false)
}
}
Err(e) => {
debug!("Failed to resize image: {}", e);
(bytes, false)
}
}
} else {
(bytes, false)
};
let file_size = bytes.len();
// Try to get image dimensions
@@ -323,7 +348,23 @@ pub async fn execute_read_image<W: UiWriter>(
.map(|(w, h)| format!("{}x{}", w, h))
.unwrap_or_else(|| "unknown".to_string());
let size_str = if file_size >= 1024 * 1024 {
let format_size = |size: usize| -> String {
if size >= 1024 * 1024 {
format!("{:.1} MB", size as f64 / (1024.0 * 1024.0))
} else if size >= 1024 {
format!("{:.1} KB", size as f64 / 1024.0)
} else {
format!("{} bytes", size)
}
};
let size_str = if was_resized {
format!(
"{}{} (resized)",
format_size(original_size),
format_size(file_size)
)
} else if file_size >= 1024 * 1024 {
format!("{:.1} MB", file_size as f64 / (1024.0 * 1024.0))
} else if file_size >= 1024 {
format!("{:.1} KB", file_size as f64 / 1024.0)
@@ -331,13 +372,16 @@ pub async fn execute_read_image<W: UiWriter>(
format!("{} bytes", file_size)
};
// If resized, the output is JPEG
let final_media_type = if was_resized { "image/jpeg" } else { media_type };
// Output imgcat inline image to terminal (height constrained)
print_imgcat(&bytes, path_str, &dim_str, media_type, &size_str, 5);
print_imgcat(&bytes, path_str, &dim_str, final_media_type, &size_str, 5);
// Store the image to be attached to the next user message
use base64::Engine;
let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes);
let image = g3_providers::ImageContent::new(media_type, encoded);
let image = g3_providers::ImageContent::new(final_media_type, encoded);
ctx.pending_images.push(image);
success_count += 1;
@@ -533,6 +577,81 @@ fn extract_path_and_content(args: &serde_json::Value) -> (Option<&str>, Option<&
}
}
/// Resize an image to be under the target size using ImageMagick.
/// Returns the resized image bytes, or the original bytes if resizing fails or isn't needed.
///
/// Uses iterative quality reduction and dimension scaling to achieve target size.
pub fn resize_image_if_needed(
bytes: &[u8],
path: &std::path::Path,
target_size: usize,
) -> std::io::Result<Vec<u8>> {
// If already under target size, return original
if bytes.len() < target_size {
return Ok(bytes.to_vec());
}
debug!(
"Image {} is {} bytes, resizing to under {} bytes",
path.display(),
bytes.len(),
target_size
);
// Create a temp file for the input
let temp_dir = std::env::temp_dir();
let input_path = temp_dir.join(format!("g3_resize_input_{}", std::process::id()));
let output_path = temp_dir.join(format!("g3_resize_output_{}.jpg", std::process::id()));
// Write input bytes to temp file
std::fs::write(&input_path, bytes)?;
// Try different quality levels, starting high and decreasing
let quality_levels = [85, 70, 55, 40, 25];
let scale_factors = [100, 80, 60, 50, 40];
for &scale in &scale_factors {
for &quality in &quality_levels {
// Use ImageMagick convert to resize
let result = std::process::Command::new("convert")
.arg(&input_path)
.arg("-resize")
.arg(format!("{}%", scale))
.arg("-quality")
.arg(format!("{}", quality))
.arg(&output_path)
.output();
if let Ok(output) = result {
if output.status.success() {
if let Ok(resized_bytes) = std::fs::read(&output_path) {
if resized_bytes.len() < target_size {
debug!(
"Resized image to {} bytes (scale={}%, quality={})",
resized_bytes.len(),
scale,
quality
);
// Clean up temp files
let _ = std::fs::remove_file(&input_path);
let _ = std::fs::remove_file(&output_path);
return Ok(resized_bytes);
}
}
}
}
}
}
// Clean up temp files
let _ = std::fs::remove_file(&input_path);
let _ = std::fs::remove_file(&output_path);
// If all attempts failed, return original bytes
debug!("Failed to resize image under target size, using original");
Ok(bytes.to_vec())
}
/// Get image dimensions from raw bytes.
pub fn get_image_dimensions(bytes: &[u8], media_type: &str) -> Option<(u32, u32)> {
match media_type {

View File

@@ -199,3 +199,32 @@ fn test_image_dimensions_gif() {
assert_eq!(width, 100);
assert_eq!(height, 200);
}
#[test]
fn test_resize_image_if_needed_small_image() {
use g3_core::tools::file_ops::resize_image_if_needed;
use std::path::Path;
// Small image should not be resized
let small_bytes = vec![0u8; 1000]; // 1KB
let path = Path::new("test.jpg");
let target_size = 5 * 1024 * 1024; // 5MB
let result = resize_image_if_needed(&small_bytes, path, target_size).unwrap();
assert_eq!(result.len(), small_bytes.len(), "Small image should not be resized");
}
#[test]
fn test_resize_image_if_needed_returns_original_on_failure() {
use g3_core::tools::file_ops::resize_image_if_needed;
use std::path::Path;
// Invalid image data - ImageMagick will fail, should return original
let invalid_bytes = vec![0u8; 6 * 1024 * 1024]; // 6MB of zeros
let path = Path::new("test.jpg");
let target_size = 5 * 1024 * 1024; // 5MB
let result = resize_image_if_needed(&invalid_bytes, path, target_size).unwrap();
// Should return original since ImageMagick can't process invalid data
assert_eq!(result.len(), invalid_bytes.len(), "Invalid image should return original");
}