Auto-resize large images (>=5MB) in read_image tool
Images >= 5MB are now automatically resized to < 4.9MB using ImageMagick before being sent to the LLM. This prevents API errors from oversized images. - Uses iterative quality/scale reduction to find optimal size - Converts to JPEG for better compression - Shows original and resized size in terminal output (e.g., '6.2 MB → 4.1 MB (resized)') - Falls back to original if ImageMagick fails or isn't available
This commit is contained in:
@@ -13,6 +13,9 @@ use crate::ToolCall;
|
|||||||
|
|
||||||
use super::executor::ToolContext;
|
use super::executor::ToolContext;
|
||||||
|
|
||||||
|
/// Maximum image size in bytes (5MB) - images larger than this will be resized
|
||||||
|
const MAX_IMAGE_SIZE: usize = 5 * 1024 * 1024;
|
||||||
|
|
||||||
/// Bytes per token heuristic (conservative estimate for code/text mix)
|
/// Bytes per token heuristic (conservative estimate for code/text mix)
|
||||||
const BYTES_PER_TOKEN: f32 = 3.5;
|
const BYTES_PER_TOKEN: f32 = 3.5;
|
||||||
|
|
||||||
@@ -313,6 +316,28 @@ pub async fn execute_read_image<W: UiWriter>(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let original_size = bytes.len();
|
||||||
|
|
||||||
|
// Resize image if it's >= 5MB (target < 4.9MB to leave margin)
|
||||||
|
let (bytes, was_resized) = if original_size >= MAX_IMAGE_SIZE {
|
||||||
|
match resize_image_if_needed(&bytes, path, MAX_IMAGE_SIZE - 100 * 1024) {
|
||||||
|
Ok(resized) => {
|
||||||
|
let resized_size = resized.len();
|
||||||
|
if resized_size < original_size {
|
||||||
|
(resized, true)
|
||||||
|
} else {
|
||||||
|
(bytes, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
debug!("Failed to resize image: {}", e);
|
||||||
|
(bytes, false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
(bytes, false)
|
||||||
|
};
|
||||||
|
|
||||||
let file_size = bytes.len();
|
let file_size = bytes.len();
|
||||||
|
|
||||||
// Try to get image dimensions
|
// Try to get image dimensions
|
||||||
@@ -323,7 +348,23 @@ pub async fn execute_read_image<W: UiWriter>(
|
|||||||
.map(|(w, h)| format!("{}x{}", w, h))
|
.map(|(w, h)| format!("{}x{}", w, h))
|
||||||
.unwrap_or_else(|| "unknown".to_string());
|
.unwrap_or_else(|| "unknown".to_string());
|
||||||
|
|
||||||
let size_str = if file_size >= 1024 * 1024 {
|
let format_size = |size: usize| -> String {
|
||||||
|
if size >= 1024 * 1024 {
|
||||||
|
format!("{:.1} MB", size as f64 / (1024.0 * 1024.0))
|
||||||
|
} else if size >= 1024 {
|
||||||
|
format!("{:.1} KB", size as f64 / 1024.0)
|
||||||
|
} else {
|
||||||
|
format!("{} bytes", size)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let size_str = if was_resized {
|
||||||
|
format!(
|
||||||
|
"{} → {} (resized)",
|
||||||
|
format_size(original_size),
|
||||||
|
format_size(file_size)
|
||||||
|
)
|
||||||
|
} else if file_size >= 1024 * 1024 {
|
||||||
format!("{:.1} MB", file_size as f64 / (1024.0 * 1024.0))
|
format!("{:.1} MB", file_size as f64 / (1024.0 * 1024.0))
|
||||||
} else if file_size >= 1024 {
|
} else if file_size >= 1024 {
|
||||||
format!("{:.1} KB", file_size as f64 / 1024.0)
|
format!("{:.1} KB", file_size as f64 / 1024.0)
|
||||||
@@ -331,13 +372,16 @@ pub async fn execute_read_image<W: UiWriter>(
|
|||||||
format!("{} bytes", file_size)
|
format!("{} bytes", file_size)
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// If resized, the output is JPEG
|
||||||
|
let final_media_type = if was_resized { "image/jpeg" } else { media_type };
|
||||||
|
|
||||||
// Output imgcat inline image to terminal (height constrained)
|
// Output imgcat inline image to terminal (height constrained)
|
||||||
print_imgcat(&bytes, path_str, &dim_str, media_type, &size_str, 5);
|
print_imgcat(&bytes, path_str, &dim_str, final_media_type, &size_str, 5);
|
||||||
|
|
||||||
// Store the image to be attached to the next user message
|
// Store the image to be attached to the next user message
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes);
|
let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes);
|
||||||
let image = g3_providers::ImageContent::new(media_type, encoded);
|
let image = g3_providers::ImageContent::new(final_media_type, encoded);
|
||||||
ctx.pending_images.push(image);
|
ctx.pending_images.push(image);
|
||||||
|
|
||||||
success_count += 1;
|
success_count += 1;
|
||||||
@@ -533,6 +577,81 @@ fn extract_path_and_content(args: &serde_json::Value) -> (Option<&str>, Option<&
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resize an image to be under the target size using ImageMagick.
|
||||||
|
/// Returns the resized image bytes, or the original bytes if resizing fails or isn't needed.
|
||||||
|
///
|
||||||
|
/// Uses iterative quality reduction and dimension scaling to achieve target size.
|
||||||
|
pub fn resize_image_if_needed(
|
||||||
|
bytes: &[u8],
|
||||||
|
path: &std::path::Path,
|
||||||
|
target_size: usize,
|
||||||
|
) -> std::io::Result<Vec<u8>> {
|
||||||
|
// If already under target size, return original
|
||||||
|
if bytes.len() < target_size {
|
||||||
|
return Ok(bytes.to_vec());
|
||||||
|
}
|
||||||
|
|
||||||
|
debug!(
|
||||||
|
"Image {} is {} bytes, resizing to under {} bytes",
|
||||||
|
path.display(),
|
||||||
|
bytes.len(),
|
||||||
|
target_size
|
||||||
|
);
|
||||||
|
|
||||||
|
// Create a temp file for the input
|
||||||
|
let temp_dir = std::env::temp_dir();
|
||||||
|
let input_path = temp_dir.join(format!("g3_resize_input_{}", std::process::id()));
|
||||||
|
let output_path = temp_dir.join(format!("g3_resize_output_{}.jpg", std::process::id()));
|
||||||
|
|
||||||
|
// Write input bytes to temp file
|
||||||
|
std::fs::write(&input_path, bytes)?;
|
||||||
|
|
||||||
|
// Try different quality levels, starting high and decreasing
|
||||||
|
let quality_levels = [85, 70, 55, 40, 25];
|
||||||
|
let scale_factors = [100, 80, 60, 50, 40];
|
||||||
|
|
||||||
|
for &scale in &scale_factors {
|
||||||
|
for &quality in &quality_levels {
|
||||||
|
// Use ImageMagick convert to resize
|
||||||
|
let result = std::process::Command::new("convert")
|
||||||
|
.arg(&input_path)
|
||||||
|
.arg("-resize")
|
||||||
|
.arg(format!("{}%", scale))
|
||||||
|
.arg("-quality")
|
||||||
|
.arg(format!("{}", quality))
|
||||||
|
.arg(&output_path)
|
||||||
|
.output();
|
||||||
|
|
||||||
|
if let Ok(output) = result {
|
||||||
|
if output.status.success() {
|
||||||
|
if let Ok(resized_bytes) = std::fs::read(&output_path) {
|
||||||
|
if resized_bytes.len() < target_size {
|
||||||
|
debug!(
|
||||||
|
"Resized image to {} bytes (scale={}%, quality={})",
|
||||||
|
resized_bytes.len(),
|
||||||
|
scale,
|
||||||
|
quality
|
||||||
|
);
|
||||||
|
// Clean up temp files
|
||||||
|
let _ = std::fs::remove_file(&input_path);
|
||||||
|
let _ = std::fs::remove_file(&output_path);
|
||||||
|
return Ok(resized_bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up temp files
|
||||||
|
let _ = std::fs::remove_file(&input_path);
|
||||||
|
let _ = std::fs::remove_file(&output_path);
|
||||||
|
|
||||||
|
// If all attempts failed, return original bytes
|
||||||
|
debug!("Failed to resize image under target size, using original");
|
||||||
|
Ok(bytes.to_vec())
|
||||||
|
}
|
||||||
|
|
||||||
/// Get image dimensions from raw bytes.
|
/// Get image dimensions from raw bytes.
|
||||||
pub fn get_image_dimensions(bytes: &[u8], media_type: &str) -> Option<(u32, u32)> {
|
pub fn get_image_dimensions(bytes: &[u8], media_type: &str) -> Option<(u32, u32)> {
|
||||||
match media_type {
|
match media_type {
|
||||||
|
|||||||
@@ -199,3 +199,32 @@ fn test_image_dimensions_gif() {
|
|||||||
assert_eq!(width, 100);
|
assert_eq!(width, 100);
|
||||||
assert_eq!(height, 200);
|
assert_eq!(height, 200);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_resize_image_if_needed_small_image() {
|
||||||
|
use g3_core::tools::file_ops::resize_image_if_needed;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
// Small image should not be resized
|
||||||
|
let small_bytes = vec![0u8; 1000]; // 1KB
|
||||||
|
let path = Path::new("test.jpg");
|
||||||
|
let target_size = 5 * 1024 * 1024; // 5MB
|
||||||
|
|
||||||
|
let result = resize_image_if_needed(&small_bytes, path, target_size).unwrap();
|
||||||
|
assert_eq!(result.len(), small_bytes.len(), "Small image should not be resized");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_resize_image_if_needed_returns_original_on_failure() {
|
||||||
|
use g3_core::tools::file_ops::resize_image_if_needed;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
// Invalid image data - ImageMagick will fail, should return original
|
||||||
|
let invalid_bytes = vec![0u8; 6 * 1024 * 1024]; // 6MB of zeros
|
||||||
|
let path = Path::new("test.jpg");
|
||||||
|
let target_size = 5 * 1024 * 1024; // 5MB
|
||||||
|
|
||||||
|
let result = resize_image_if_needed(&invalid_bytes, path, target_size).unwrap();
|
||||||
|
// Should return original since ImageMagick can't process invalid data
|
||||||
|
assert_eq!(result.len(), invalid_bytes.len(), "Invalid image should return original");
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user