webui: support video files as input (#22830)

tools/ui/src/lib/utils/attachment-type.ts

@@ -103,3 +103,24 @@ export function isAudioFile(
 
 	return false;
 }
+
+/**
+ * Determines if an attachment or uploaded file is a video file
+ * @param uploadedFile - Optional uploaded file
+ * @param attachment - Optional database attachment
+ * @returns true if the file is a video file
+ */
+export function isVideoFile(
+	attachment?: DatabaseMessageExtra,
+	uploadedFile?: ChatUploadedFile
+): boolean {
+	if (uploadedFile) {
+		return getUploadedFileCategory(uploadedFile) === FileTypeCategory.VIDEO;
+	}
+
+	if (attachment) {
+		return attachment.type === AttachmentType.VIDEO;
+	}
+
+	return false;
+}

tools/ui/src/lib/utils/convert-files-to-extra.ts

@@ -89,6 +89,21 @@ export async function parseFilesToMessageExtras(
 			} catch (error) {
 				console.error(`Failed to process audio file ${file.name}:`, error);
 			}
+		} else if (getFileTypeCategory(file.type) === FileTypeCategory.VIDEO) {
+			// Process video files (MP4, etc)
+			try {
+				const base64Data = await readFileAsBase64(file.file);
+
+				extras.push({
+					type: AttachmentType.VIDEO,
+					name: file.name,
+					size: file.size,
+					base64Data: base64Data,
+					mimeType: file.type
+				});
+			} catch (error) {
+				console.error(`Failed to process video file ${file.name}:`, error);
+			}
 		} else if (getFileTypeCategory(file.type) === FileTypeCategory.PDF) {
 			try {
 				// Always get base64 data for preview functionality

tools/ui/src/lib/utils/file-type.ts

@@ -1,5 +1,6 @@
 import {
 	AUDIO_FILE_TYPES,
+	VIDEO_FILE_TYPES,
 	IMAGE_FILE_TYPES,
 	PDF_FILE_TYPES,
 	TEXT_FILE_TYPES
@@ -12,6 +13,7 @@ import {
 	FileTypeCategory,
 	MimeTypeApplication,
 	MimeTypeAudio,
+	MimeTypeVideo,
 	MimeTypeImage,
 	MimeTypeText
 } from '$lib/enums';
@@ -35,6 +37,11 @@ export function getFileTypeCategory(mimeType: string): FileTypeCategory | null {
 		case MimeTypeAudio.WEBM_OPUS:
 			return FileTypeCategory.AUDIO;
 
+		// Video
+		case MimeTypeVideo.MP4:
+		case MimeTypeVideo.OGG:
+			return FileTypeCategory.VIDEO;
+
 		// PDF
 		case MimeTypeApplication.PDF:
 			return FileTypeCategory.PDF;
@@ -179,6 +186,12 @@ export function getFileTypeByExtension(filename: string): string | null {
 		}
 	}
 
+	for (const [key, type] of Object.entries(VIDEO_FILE_TYPES)) {
+		if ((type.extensions as readonly string[]).includes(extension)) {
+			return `${FileTypeCategory.VIDEO}:${key}`;
+		}
+	}
+
 	for (const [key, type] of Object.entries(PDF_FILE_TYPES)) {
 		if ((type.extensions as readonly string[]).includes(extension)) {
 			return `${FileTypeCategory.PDF}:${key}`;

tools/ui/src/lib/utils/index.ts

@@ -14,7 +14,7 @@ export { validateApiKey } from './api-key-validation';
 
 // Attachment utilities
 export { getAttachmentDisplayItems, isMcpPrompt, isMcpResource } from './attachment-display';
-export { isTextFile, isImageFile, isPdfFile, isAudioFile } from './attachment-type';
+export { isTextFile, isImageFile, isPdfFile, isAudioFile, isVideoFile } from './attachment-type';
 
 // Textarea utilities
 export { default as autoResizeTextarea } from './autoresize-textarea';

tools/ui/src/lib/utils/modality-file-validation.ts

@@ -45,6 +45,10 @@ export function isFileTypeSupportedByModel(
 			// Audio files require audio support
 			return capabilities.hasAudio;
 
+		case FileTypeCategory.VIDEO:
+			// Video files require video support
+			return capabilities.hasVideo;
+
 		default:
 			// Unknown categories - be conservative and allow
 			return true;
@@ -69,7 +73,7 @@ export function filterFilesByModalities(
 	const unsupportedFiles: File[] = [];
 	const modalityReasons: Record<string, string> = {};
 
-	const { hasVision, hasAudio } = capabilities;
+	const { hasVision, hasAudio, hasVideo } = capabilities;
 
 	for (const file of files) {
 		const category = getFileTypeCategory(file.type);
@@ -91,6 +95,13 @@ export function filterFilesByModalities(
 				}
 				break;
 
+			case FileTypeCategory.VIDEO:
+				if (!hasVideo) {
+					isSupported = false;
+					reason = 'Video files require a video-capable model';
+				}
+				break;
+
 			case FileTypeCategory.TEXT:
 			case FileTypeCategory.PDF:
 				// Always supported
@@ -127,7 +138,7 @@ export function generateModalityErrorMessage(
 ): string {
 	if (unsupportedFiles.length === 0) return '';
 
-	const { hasVision, hasAudio } = capabilities;
+	const { hasVision, hasAudio, hasVideo } = capabilities;
 
 	let message = '';
 
@@ -144,6 +155,7 @@ export function generateModalityErrorMessage(
 	const supportedTypes: string[] = ['text files', 'PDFs'];
 	if (hasVision) supportedTypes.push('images');
 	if (hasAudio) supportedTypes.push('audio files');
+	if (hasVideo) supportedTypes.push('video files');
 
 	message += ` This model supports: ${supportedTypes.join(', ')}.`;
 

tools/ui/src/lib/utils/process-uploaded-files.ts

@@ -117,6 +117,10 @@ export async function processFilesToChatUploaded(
 				// Generate preview URL for audio files
 				const preview = await readFileAsDataURL(file);
 				results.push({ ...base, preview });
+			} else if (getFileTypeCategory(file.type) === FileTypeCategory.VIDEO) {
+				// Generate preview URL for video files
+				const preview = await readFileAsDataURL(file);
+				results.push({ ...base, preview });
 			} else {
 				// Fallback: treat unknown files as text
 				try {