fix: auto-resize images exceeding 1568px dimension to prevent 413 Payload Too Large
The Anthropic API was rejecting requests with multiple high-resolution images (~2000x3000 pixels each) even though individual file sizes were under limits. Root cause: Code only checked per-image file size (3.75MB), not dimensions. Claude recommends images ≤1568px on longest edge and has 32MB total request limit. Changes: - Add MAX_IMAGE_DIMENSION (1568px) and MAX_TOTAL_IMAGE_PAYLOAD (20MB) constants - Trigger resize when dimensions > 1568px (not just file size > 3.75MB) - Add new resize_image_to_dimensions() for dimension-constrained resizing - Track cumulative payload size across multiple images - Warn if total payload exceeds recommended limit Test results with Walking Dead comic images: - WD_0001_0001.jpg: 800KB 1987x3057 → 321KB 1019x1568 - WD_0001_1064.png: 150KB 1988x3057 → 143KB 1020x1568 - WD_0002_0001.jpg: 1023KB 1988x3056 → 292KB 1020x1568 - Total payload: ~2.5MB → ~1MB base64
This commit is contained in:
@@ -19,6 +19,15 @@ const MAX_BASE64_SIZE: usize = 5 * 1024 * 1024;
|
|||||||
/// Maximum raw image size before base64 encoding (~3.75MB to stay under 5MB after encoding)
|
/// Maximum raw image size before base64 encoding (~3.75MB to stay under 5MB after encoding)
|
||||||
const MAX_IMAGE_SIZE: usize = (MAX_BASE64_SIZE * 3) / 4;
|
const MAX_IMAGE_SIZE: usize = (MAX_BASE64_SIZE * 3) / 4;
|
||||||
|
|
||||||
|
/// Maximum recommended image dimension (longest edge) for optimal API performance.
|
||||||
|
/// Images larger than this are auto-scaled by Claude anyway, wasting bandwidth.
|
||||||
|
/// Per Anthropic docs: "We recommend resizing images to no more than 1.15 megapixels"
|
||||||
|
const MAX_IMAGE_DIMENSION: u32 = 1568;
|
||||||
|
|
||||||
|
/// Maximum total payload size for all images combined (leave room for context).
|
||||||
|
/// Anthropic's limit is 32MB total request size; we target 20MB for images to leave headroom.
|
||||||
|
const MAX_TOTAL_IMAGE_PAYLOAD: usize = 20 * 1024 * 1024;
|
||||||
|
|
||||||
/// Bytes per token heuristic (conservative estimate for code/text mix)
|
/// Bytes per token heuristic (conservative estimate for code/text mix)
|
||||||
const BYTES_PER_TOKEN: f32 = 3.5;
|
const BYTES_PER_TOKEN: f32 = 3.5;
|
||||||
|
|
||||||
@@ -280,6 +289,7 @@ pub async fn execute_read_image<W: UiWriter>(
|
|||||||
|
|
||||||
let mut results: Vec<String> = Vec::new();
|
let mut results: Vec<String> = Vec::new();
|
||||||
let mut success_count = 0;
|
let mut success_count = 0;
|
||||||
|
let mut cumulative_payload_size: usize = 0;
|
||||||
|
|
||||||
// Print └─ and newline before images to break out of tool output box
|
// Print └─ and newline before images to break out of tool output box
|
||||||
println!("└─\n");
|
println!("└─\n");
|
||||||
@@ -321,16 +331,33 @@ pub async fn execute_read_image<W: UiWriter>(
|
|||||||
|
|
||||||
let original_size = bytes.len();
|
let original_size = bytes.len();
|
||||||
|
|
||||||
// Resize image if it exceeds MAX_IMAGE_SIZE (~3.75MB raw = ~5MB base64)
|
// Get dimensions early to decide if we need to resize
|
||||||
// Target slightly smaller to leave margin for base64 overhead
|
let original_dimensions = get_image_dimensions(&bytes, media_type);
|
||||||
let (bytes, was_resized) = if original_size >= MAX_IMAGE_SIZE {
|
|
||||||
match resize_image_if_needed(&bytes, path, MAX_IMAGE_SIZE - 150 * 1024) {
|
// Determine if resize is needed based on:
|
||||||
|
// 1. Dimensions exceed MAX_IMAGE_DIMENSION (1568px) - Claude auto-scales anyway
|
||||||
|
// 2. File size exceeds MAX_IMAGE_SIZE (~3.75MB)
|
||||||
|
// 3. Adding this image would exceed cumulative payload limit
|
||||||
|
let needs_resize = original_size >= MAX_IMAGE_SIZE
|
||||||
|
|| original_dimensions
|
||||||
|
.map(|(w, h)| w > MAX_IMAGE_DIMENSION || h > MAX_IMAGE_DIMENSION)
|
||||||
|
.unwrap_or(false)
|
||||||
|
|| (cumulative_payload_size + (original_size * 4 / 3)) > MAX_TOTAL_IMAGE_PAYLOAD;
|
||||||
|
|
||||||
|
let (bytes, was_resized) = if needs_resize {
|
||||||
|
// Calculate target size: either fit under per-image limit or leave room in cumulative budget
|
||||||
|
let remaining_budget = MAX_TOTAL_IMAGE_PAYLOAD.saturating_sub(cumulative_payload_size);
|
||||||
|
let target_raw_size = (remaining_budget * 3 / 4).min(MAX_IMAGE_SIZE - 150 * 1024);
|
||||||
|
|
||||||
|
match resize_image_to_dimensions(&bytes, path, MAX_IMAGE_DIMENSION, target_raw_size) {
|
||||||
Ok(resized) => {
|
Ok(resized) => {
|
||||||
let resized_size = resized.len();
|
let resized_size = resized.len();
|
||||||
if resized_size < original_size {
|
if resized_size < original_size {
|
||||||
(resized, true)
|
(resized, true)
|
||||||
} else {
|
} else {
|
||||||
(bytes, false)
|
// Resize didn't help, use original but warn if it's huge
|
||||||
|
debug!("Resize didn't reduce size, using original");
|
||||||
|
(bytes, original_dimensions.map(|(w, h)| w > MAX_IMAGE_DIMENSION || h > MAX_IMAGE_DIMENSION).unwrap_or(false))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -344,8 +371,8 @@ pub async fn execute_read_image<W: UiWriter>(
|
|||||||
|
|
||||||
let file_size = bytes.len();
|
let file_size = bytes.len();
|
||||||
|
|
||||||
// Try to get image dimensions
|
// Get final dimensions (may have changed if resized)
|
||||||
let dimensions = get_image_dimensions(&bytes, media_type);
|
let dimensions = if was_resized { get_image_dimensions(&bytes, "image/jpeg") } else { original_dimensions };
|
||||||
|
|
||||||
// Build info string
|
// Build info string
|
||||||
let dim_str = dimensions
|
let dim_str = dimensions
|
||||||
@@ -385,6 +412,18 @@ pub async fn execute_read_image<W: UiWriter>(
|
|||||||
// Store the image to be attached to the next user message
|
// Store the image to be attached to the next user message
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes);
|
let encoded = base64::engine::general_purpose::STANDARD.encode(&bytes);
|
||||||
|
let encoded_size = encoded.len();
|
||||||
|
|
||||||
|
// Track cumulative payload and warn if approaching limit
|
||||||
|
cumulative_payload_size += encoded_size;
|
||||||
|
if cumulative_payload_size > MAX_TOTAL_IMAGE_PAYLOAD {
|
||||||
|
results.push(format!(
|
||||||
|
"⚠️ Warning: Total image payload ({:.1} MB) exceeds recommended limit ({:.1} MB). Request may fail.",
|
||||||
|
cumulative_payload_size as f64 / (1024.0 * 1024.0),
|
||||||
|
MAX_TOTAL_IMAGE_PAYLOAD as f64 / (1024.0 * 1024.0)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
let image = g3_providers::ImageContent::new(final_media_type, encoded);
|
let image = g3_providers::ImageContent::new(final_media_type, encoded);
|
||||||
ctx.pending_images.push(image);
|
ctx.pending_images.push(image);
|
||||||
|
|
||||||
@@ -581,6 +620,79 @@ fn extract_path_and_content(args: &serde_json::Value) -> (Option<&str>, Option<&
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resize an image to fit within max_dimension pixels (longest edge) and target_size bytes.
|
||||||
|
/// This is the primary resize function that handles both dimension and size constraints.
|
||||||
|
///
|
||||||
|
/// Uses ImageMagick to:
|
||||||
|
/// 1. First resize to fit within max_dimension (if needed)
|
||||||
|
/// 2. Then reduce quality/scale to fit within target_size (if needed)
|
||||||
|
pub fn resize_image_to_dimensions(
|
||||||
|
bytes: &[u8],
|
||||||
|
path: &std::path::Path,
|
||||||
|
max_dimension: u32,
|
||||||
|
target_size: usize,
|
||||||
|
) -> std::io::Result<Vec<u8>> {
|
||||||
|
debug!(
|
||||||
|
"Resizing image {} to max {}px and under {} bytes",
|
||||||
|
path.display(),
|
||||||
|
max_dimension,
|
||||||
|
target_size
|
||||||
|
);
|
||||||
|
|
||||||
|
// Create temp files for processing
|
||||||
|
let temp_dir = std::env::temp_dir();
|
||||||
|
let input_path = temp_dir.join(format!("g3_resize_input_{}", std::process::id()));
|
||||||
|
let output_path = temp_dir.join(format!("g3_resize_output_{}.jpg", std::process::id()));
|
||||||
|
|
||||||
|
// Write input bytes to temp file
|
||||||
|
std::fs::write(&input_path, bytes)?;
|
||||||
|
|
||||||
|
// Quality levels to try (start high for best quality)
|
||||||
|
let quality_levels = [90, 80, 70, 60, 50, 40];
|
||||||
|
|
||||||
|
for &quality in &quality_levels {
|
||||||
|
// Use ImageMagick to resize: constrain to max_dimension on longest edge
|
||||||
|
// The "WxH>" syntax means "resize only if larger, maintain aspect ratio"
|
||||||
|
let resize_spec = format!("{}x{}>", max_dimension, max_dimension);
|
||||||
|
|
||||||
|
let result = std::process::Command::new("convert")
|
||||||
|
.arg(&input_path)
|
||||||
|
.arg("-resize")
|
||||||
|
.arg(&resize_spec)
|
||||||
|
.arg("-quality")
|
||||||
|
.arg(format!("{}", quality))
|
||||||
|
.arg(&output_path)
|
||||||
|
.output();
|
||||||
|
|
||||||
|
if let Ok(output) = result {
|
||||||
|
if output.status.success() {
|
||||||
|
if let Ok(resized_bytes) = std::fs::read(&output_path) {
|
||||||
|
if resized_bytes.len() <= target_size {
|
||||||
|
debug!(
|
||||||
|
"Resized image to {} bytes (max_dim={}, quality={})",
|
||||||
|
resized_bytes.len(),
|
||||||
|
max_dimension,
|
||||||
|
quality
|
||||||
|
);
|
||||||
|
// Clean up temp files
|
||||||
|
let _ = std::fs::remove_file(&input_path);
|
||||||
|
let _ = std::fs::remove_file(&output_path);
|
||||||
|
return Ok(resized_bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up temp files
|
||||||
|
let _ = std::fs::remove_file(&input_path);
|
||||||
|
let _ = std::fs::remove_file(&output_path);
|
||||||
|
|
||||||
|
// If all attempts failed, return original bytes
|
||||||
|
debug!("Failed to resize image to target constraints, using original");
|
||||||
|
Ok(bytes.to_vec())
|
||||||
|
}
|
||||||
|
|
||||||
/// Resize an image to be under the target size using ImageMagick.
|
/// Resize an image to be under the target size using ImageMagick.
|
||||||
/// Returns the resized image bytes, or the original bytes if resizing fails or isn't needed.
|
/// Returns the resized image bytes, or the original bytes if resizing fails or isn't needed.
|
||||||
///
|
///
|
||||||
|
|||||||
Reference in New Issue
Block a user