ui: Add max image size option (#22849)

* webui: Add max image size option

* remove magic numbers

* support all image formats

* use const

* Move regex to match b64 images to constants

* use SETTINGS_KEYS to get max image resolution setting

* Do not touch the image if already under the size threshold

tools/ui/src/lib/constants/image-size.ts

@@ -0,0 +1 @@
+export const MEGAPIXELS_TO_PIXELS = 1_000_000;

tools/ui/src/lib/constants/settings-keys.ts

@@ -18,6 +18,7 @@ export const SETTINGS_KEYS = {
 	TITLE_GENERATION_USE_FIRST_LINE: 'titleGenerationUseFirstLine',
 	TITLE_GENERATION_USE_LLM: 'titleGenerationUseLLM',
 	TITLE_GENERATION_PROMPT: 'titleGenerationPrompt',
+	MAX_IMAGE_RESOLUTION: 'maxImageMPixels',
 	// Display
 	SHOW_MESSAGE_STATS: 'showMessageStats',
 	SHOW_THOUGHT_IN_PROGRESS: 'showThoughtInProgress',

tools/ui/src/lib/constants/settings-registry.ts

@@ -193,6 +193,14 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
 				defaultValue: TITLE_GENERATION.DEFAULT_PROMPT,
 				type: SettingsFieldType.TEXTAREA,
 				section: SETTINGS_SECTION_SLUGS.GENERAL
+			},
+			{
+				key: SETTINGS_KEYS.MAX_IMAGE_RESOLUTION,
+				label: 'Maximum image resolution (megapixels)',
+				help: 'Images larger than this will be resized before sending to server. Set to 0 to disable.',
+				defaultValue: 0,
+				type: SettingsFieldType.INPUT,
+				section: SETTINGS_SECTION_SLUGS.GENERAL
 			}
 		]
 	},

tools/ui/src/lib/constants/uri-template.ts

@@ -55,3 +55,6 @@ export const VARIABLE_PREFIX_MODIFIER_REGEX = /:[\d]+$/;
 
 /** Regex to strip one or more leading slashes */
 export const LEADING_SLASHES_REGEX = /^\/+/;
+
+/** Regex to match base64-encoded image URIs (format: "data:image/[media type];base64,[data]")*/
+export const BASE64_IMAGE_URI_REGEX = /^data:(image\/[a-z0-9.\-+]+);base64/;

tools/ui/src/lib/services/chat.service.ts

@@ -5,7 +5,8 @@ import {
 	ATTACHMENT_LABEL_PDF_FILE,
 	ATTACHMENT_LABEL_MCP_PROMPT,
 	ATTACHMENT_LABEL_MCP_RESOURCE,
-	LEGACY_AGENTIC_REGEX
+	LEGACY_AGENTIC_REGEX,
+	SETTINGS_KEYS
 } from '$lib/constants';
 import {
 	AttachmentType,
@@ -27,6 +28,9 @@ import type {
 	DatabaseMessageExtraMcpResource
 } from '$lib/types';
 import { modelsStore } from '$lib/stores/models.svelte';
+import { settingsStore } from '../stores/settings.svelte';
+import { capImageDataURLSize } from '../utils/cap-img-size';
+import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size';
 
 function getAudioInputFormat(mimeType: string): AudioInputFormat {
 	const normalizedMimeType = mimeType.trim().toLowerCase();
@@ -156,26 +160,28 @@ export class ChatService {
 			continueFinalMessage
 		} = options;
 
-		const normalizedMessages: ApiChatMessageData[] = messages
+		const normalizedMessages: ApiChatMessageData[] = (
-			.map((msg) => {
+			await Promise.all(
-				if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
+				messages.map((msg) => {
-					const dbMsg = msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] };
+					if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
+						const dbMsg = msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] };
 
-					return ChatService.convertDbMessageToApiChatMessageData(dbMsg);
+						return ChatService.convertDbMessageToApiChatMessageData(dbMsg);
-				} else {
+					} else {
-					return msg as ApiChatMessageData;
+						return msg as ApiChatMessageData;
-				}
+					}
-			})
+				})
-			.filter((msg) => {
+			)
-				// Filter out empty system messages
+		).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => {
-				if (msg.role === MessageRole.SYSTEM) {
+			// Filter out empty system messages
-					const content = typeof msg.content === 'string' ? msg.content : '';
+			if (msg.role === MessageRole.SYSTEM) {
+				const content = typeof msg.content === 'string' ? msg.content : '';
 
-					return content.trim().length > 0;
+				return content.trim().length > 0;
-				}
+			}
 
-				return true;
+			return true;
-			});
+		});
 
 		// Filter out image attachments if the model doesn't support vision
 		if (options.model && !modelsStore.modelSupportsVision(options.model)) {
@@ -404,25 +410,27 @@ export class ChatService {
 		excludeReasoning?: boolean,
 		signal?: AbortSignal
 	): Promise<void> {
-		const normalizedMessages: ApiChatMessageData[] = messages
+		const normalizedMessages: ApiChatMessageData[] = (
-			.map((msg) => {
+			await Promise.all(
-				if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
+				messages.map((msg) => {
-					return ChatService.convertDbMessageToApiChatMessageData(
+					if ('id' in msg && 'convId' in msg && 'timestamp' in msg) {
-						msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
+						return ChatService.convertDbMessageToApiChatMessageData(
-					);
+							msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
-				}
+						);
+					}
 
-				return msg as ApiChatMessageData;
+					return msg as ApiChatMessageData;
-			})
+				})
-			.filter((msg) => {
+			)
-				if (msg.role === MessageRole.SYSTEM) {
+		).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => {
-					const content = typeof msg.content === 'string' ? msg.content : '';
+			if (msg.role === MessageRole.SYSTEM) {
+				const content = typeof msg.content === 'string' ? msg.content : '';
 
-					return content.trim().length > 0;
+				return content.trim().length > 0;
-				}
+			}
 
-				return true;
+			return true;
-			});
+		});
 
 		const requestBody: Record<string, unknown> = {
 			messages: normalizedMessages.map((msg: ApiChatMessageData) => {
@@ -805,9 +813,9 @@ export class ChatService {
 	 * @returns {ApiChatMessageData} object formatted for the chat completion API
 	 * @static
 	 */
-	static convertDbMessageToApiChatMessageData(
+	static async convertDbMessageToApiChatMessageData(
 		message: DatabaseMessage & { extra?: DatabaseMessageExtra[] }
-	): ApiChatMessageData {
+	): Promise<ApiChatMessageData> {
 		// Handle tool result messages (role: 'tool')
 		if (message.role === MessageRole.TOOL && message.toolCallId) {
 			return {
@@ -885,9 +893,14 @@ export class ChatService {
 		);
 
 		for (const image of imageFiles) {
+			const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION);
+			let base64Url = image.base64Url;
+			if (maxImageResolution > 1 / MEGAPIXELS_TO_PIXELS) {
+				base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution);
+			}
 			contentParts.push({
 				type: ContentPartType.IMAGE_URL,
-				image_url: { url: image.base64Url }
+				image_url: { url: base64Url }
 			});
 		}
 

tools/ui/src/lib/stores/agentic.svelte.ts

@@ -416,21 +416,23 @@ class AgenticStore {
 
 		console.log(`[AgenticStore] Starting agentic flow with ${tools.length} tools`);
 
-		const normalizedMessages: ApiChatMessageData[] = messages
+		const normalizedMessages: ApiChatMessageData[] = (
-			.map((msg) => {
+			await Promise.all(
-				if ('id' in msg && 'convId' in msg && 'timestamp' in msg)
+				messages.map((msg) => {
-					return ChatService.convertDbMessageToApiChatMessageData(
+					if ('id' in msg && 'convId' in msg && 'timestamp' in msg)
-						msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
+						return ChatService.convertDbMessageToApiChatMessageData(
-					);
+							msg as DatabaseMessage & { extra?: DatabaseMessageExtra[] }
-				return msg as ApiChatMessageData;
+						);
-			})
+					return msg as ApiChatMessageData;
-			.filter((msg) => {
+				})
-				if (msg.role === MessageRole.SYSTEM) {
+			)
-					const content = typeof msg.content === 'string' ? msg.content : '';
+		).filter((msg: { role: ChatRole; content: string | ApiChatMessageContentPart[] }) => {
-					return content.trim().length > 0;
+			if (msg.role === MessageRole.SYSTEM) {
-				}
+				const content = typeof msg.content === 'string' ? msg.content : '';
-				return true;
+				return content.trim().length > 0;
-			});
+			}
+			return true;
+		});
 
 		this.updateSession(conversationId, {
 			isRunning: true,

tools/ui/src/lib/utils/cap-img-size.ts

@@ -0,0 +1,73 @@
+import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size';
+import { BASE64_IMAGE_URI_REGEX } from '$lib/constants/uri-template';
+import { MimeTypeImage } from '$lib/enums';
+
+/**
+ * Converts an Image base64 data URL to another Image data URL with capped dimensions to reduce file size.
+ * @param base64UrlImage - The Image base64 data URL to convert
+ * @param maxMegapixels - The maximum image size in megapixels for the output Image
+ * @returns Promise resolving to Image data URL
+ */
+export function capImageDataURLSize(
+	base64UrlImage: string,
+	maxMegapixels: number
+): Promise<string> {
+	return new Promise((resolve, reject) => {
+		try {
+			
+			const mimeMatch = base64UrlImage.match(BASE64_IMAGE_URI_REGEX);
+			
+			if (!mimeMatch) {
+				return reject(new Error('Invalid data URL format.'));
+			}
+
+			const mimeType = mimeMatch[1] as MimeTypeImage;
+
+			if (!Object.values(MimeTypeImage).includes(mimeType)) {
+				return reject(new Error(`Unsupported image MIME type: ${mimeType}`));
+			}
+
+			const img = new Image();
+
+			img.onload = () => {
+				try {
+					const canvas = document.createElement('canvas');
+					const ctx = canvas.getContext('2d');
+
+					if (!ctx) {
+						throw new Error('Failed to get 2D canvas context.');
+					}
+
+					const targetWidth = img.naturalWidth;
+					const targetHeight = img.naturalHeight;
+					const totalPixels = targetWidth * targetHeight;
+					const maxPixels = Math.floor(maxMegapixels * MEGAPIXELS_TO_PIXELS);
+
+					if (maxPixels > 0 && totalPixels > maxPixels) {
+						const scaleFactor = Math.sqrt(maxPixels / totalPixels);
+						canvas.width = Math.floor(targetWidth * scaleFactor);
+						canvas.height = Math.floor(targetHeight * scaleFactor);
+					} else {
+						return resolve(base64UrlImage);
+					}
+
+					ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+					resolve(canvas.toDataURL(mimeType));
+				} catch (err) {
+					reject(err instanceof Error ? err : new Error(String(err)));
+				}
+			};
+
+			img.onerror = () => {
+				reject(new Error('Failed to load image.'));
+			};
+
+			img.src = base64UrlImage;
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error);
+			const errorMessage = `Error resizing image: ${message}`;
+			console.error(errorMessage, error);
+			reject(new Error(errorMessage));
+		}
+	});
+}