webui: support video files as input (#22830)

2026-05-17 02:13:44 +02:00
parent b64739ea39
commit 4f13cb7424
36 changed files with 310 additions and 28 deletions
@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { Eye, Mic } from '@lucide/svelte';
+	import { Eye, Mic, Video } from '@lucide/svelte';
 	import { ModelModality } from '$lib/enums';

 	interface Props {
@@ -11,7 +11,7 @@
 </script>

 {#each modalities as modality (modality)}
-	{#if modality === ModelModality.VISION || modality === ModelModality.AUDIO}
+	{#if modality === ModelModality.VISION || modality === ModelModality.AUDIO || modality === ModelModality.VIDEO}
 		<span
 			class={[
 				'inline-flex items-center gap-1 rounded-md bg-muted px-2 py-1 text-xs font-medium',
@@ -21,7 +21,11 @@
 			{#if modality === ModelModality.VISION}
 				<Eye class="h-3 w-3" />

-				Vision
+				Vision (Image)
+			{:else if modality === ModelModality.VIDEO}
+				<Video class="h-3 w-3" />
+
+				Vision (Video)
 			{:else}
 				<Mic class="h-3 w-3" />

@@ -1,10 +1,12 @@
 <script lang="ts">
-	import { X } from '@lucide/svelte';
+	import { X, Music, Video } from '@lucide/svelte';
 	import {
 		formatFileSize,
 		getFileTypeLabel,
 		getPreviewText,
 		isPdfFile,
+		isAudioFile,
+		isVideoFile,
 		isTextFile
 	} from '$lib/utils';
 	import { ActionIcon } from '$lib/components/app';
@@ -38,6 +40,8 @@
 	}: Props = $props();

 	let isPdf = $derived(isPdfFile(attachment, uploadedFile));
+	let isAudio = $derived(isAudioFile(attachment, uploadedFile));
+	let isVideo = $derived(isVideoFile(attachment, uploadedFile));
 	let isPdfWithContent = $derived(isPdf && !!textContent);

 	let isText = $derived(isTextFile(attachment, uploadedFile));
@@ -102,7 +106,13 @@
 	<div
 		class="flex h-8 w-8 items-center justify-center rounded bg-primary/10 text-xs font-medium text-primary"
 	>
-		{fileTypeLabel}
+		{#if isAudio}
+			<Music class="h-4 w-4 text-white/70" />
+		{:else if isVideo}
+			<Video class="h-4 w-4 text-white/70" />
+		{:else}
+			{fileTypeLabel}
+		{/if}
 	</div>
 {/snippet}

@@ -12,6 +12,7 @@
 		getAttachmentDisplayItems,
 		getLanguageFromFilename,
 		isAudioFile,
+		isVideoFile,
 		isImageFile,
 		isMcpPrompt,
 		isMcpResource,
@@ -29,6 +30,7 @@
 		textContent?: string;
 		isImage: boolean;
 		isAudio: boolean;
+		isVideo: boolean;
 	}

 	interface Props {
@@ -54,7 +56,8 @@
 				(item): PreviewItem => ({
 					...item,
 					isImage: isImageFile(item.attachment, item.uploadedFile),
-					isAudio: isAudioFile(item.attachment, item.uploadedFile)
+					isAudio: isAudioFile(item.attachment, item.uploadedFile),
+					isVideo: isVideoFile(item.attachment, item.uploadedFile)
 				})
 			)
 	);
@@ -102,6 +105,9 @@
 	let isAudio = $derived(
 		currentItem ? isAudioFile(currentItem.attachment, currentItem.uploadedFile) : false
 	);
+	let isVideo = $derived(
+		currentItem ? isVideoFile(currentItem.attachment, currentItem.uploadedFile) : false
+	);
 	let isImage = $derived(
 		currentItem ? isImageFile(currentItem.attachment, currentItem.uploadedFile) : false
 	);
@@ -148,6 +154,20 @@
 			: null
 	);

+	let videoSrc = $derived(
+		isVideo && currentItem
+			? (currentItem.uploadedFile?.preview ??
+					(currentItem.attachment &&
+					'mimeType' in currentItem.attachment &&
+					'base64Data' in currentItem.attachment
+						? createBase64DataUrl(
+								currentItem.attachment.mimeType,
+								currentItem.attachment.base64Data
+							)
+						: null))
+			: null
+	);
+
 	export function prev() {
 		currentIndex = currentIndex > 0 ? currentIndex - 1 : allItems.length - 1;
 	}
@@ -173,11 +193,13 @@
 					{currentItem}
 					{isImage}
 					{isAudio}
+					{isVideo}
 					{isPdf}
 					{isText}
 					{displayPreview}
 					{displayTextContent}
 					{audioSrc}
+					{videoSrc}
 					{language}
 					{hasVisionModality}
 					{activeModelId}
@@ -1,9 +1,10 @@
 <script lang="ts">
 	import type { ChatAttachmentDisplayItem } from '$lib/types';
-	import { Image, Music, FileText, FileIcon } from '@lucide/svelte';
+	import { Image, Music, Video, FileText, FileIcon } from '@lucide/svelte';
 	import ChatAttachmentsPreviewCurrentItemPdf from './ChatAttachmentsPreviewCurrentItemPdf.svelte';
 	import ChatAttachmentsPreviewCurrentItemImage from './ChatAttachmentsPreviewCurrentItemImage.svelte';
 	import ChatAttachmentsPreviewCurrentItemAudio from './ChatAttachmentsPreviewCurrentItemAudio.svelte';
+	import ChatAttachmentsPreviewCurrentItemVideo from './ChatAttachmentsPreviewCurrentItemVideo.svelte';
 	import ChatAttachmentsPreviewCurrentItemText from './ChatAttachmentsPreviewCurrentItemText.svelte';
 	import ChatAttachmentsPreviewCurrentItemUnavailable from './ChatAttachmentsPreviewCurrentItemUnavailable.svelte';

@@ -11,11 +12,13 @@
 		currentItem: ChatAttachmentDisplayItem | null;
 		isImage: boolean;
 		isAudio: boolean;
+		isVideo: boolean;
 		isPdf: boolean;
 		isText: boolean;
 		displayPreview: string | undefined;
 		displayTextContent: string | undefined;
 		audioSrc: string | null;
+		videoSrc: string | null;
 		language: string;
 		hasVisionModality: boolean;
 		activeModelId?: string;
@@ -25,21 +28,25 @@
 		currentItem,
 		isImage,
 		isAudio,
+		isVideo,
 		isPdf,
 		isText,
 		displayPreview,
 		displayTextContent,
 		audioSrc,
+		videoSrc,
 		language,
 		hasVisionModality,
 		activeModelId
 	}: Props = $props();

 	let IconComponent = $derived(
-		isImage ? Image : isText || isPdf ? FileText : isAudio ? Music : FileIcon
+		isImage ? Image : isText || isPdf ? FileText : isAudio ? Music : isVideo ? Video : FileIcon
 	);

-	let isUnavailable = $derived(!isPdf && !isImage && !(isText && displayTextContent) && !isAudio);
+	let isUnavailable = $derived(
+		!isPdf && !isImage && !(isText && displayTextContent) && !isAudio && !isVideo
+	);
 </script>

 {#if currentItem}
@@ -58,6 +65,8 @@
 			<ChatAttachmentsPreviewCurrentItemText {displayTextContent} {language} />
 		{:else if isAudio}
 			<ChatAttachmentsPreviewCurrentItemAudio {currentItem} {audioSrc} />
+		{:else if isVideo}
+			<ChatAttachmentsPreviewCurrentItemVideo {currentItem} {videoSrc} />
 		{:else if isUnavailable}
 			<ChatAttachmentsPreviewCurrentItemUnavailable {IconComponent} />
 		{/if}
@@ -0,0 +1,26 @@
+<script lang="ts">
+	import { Video } from '@lucide/svelte';
+
+	interface Props {
+		currentItem: { name?: string } | null;
+		videoSrc: string | null;
+	}
+
+	let { currentItem, videoSrc }: Props = $props();
+</script>
+
+<div class="flex flex-1 items-center justify-center p-8">
+	<div class="w-full max-w-md text-center">
+		<Video class="mx-auto mb-4 h-16 w-16 text-white/50" />
+
+		{#if videoSrc}
+			<video controls class="mb-4 w-full" src={videoSrc}>
+				Your browser does not support the video element.
+			</video>
+		{:else}
+			<p class="mb-4 text-white/70">Video preview not available</p>
+		{/if}
+
+		<p class="text-sm text-white/50">{currentItem?.name || 'Video'}</p>
+	</div>
+</div>
@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { Music, FileText } from '@lucide/svelte';
+	import { Music, Video, FileText } from '@lucide/svelte';
 	import { HorizontalScrollCarousel } from '$lib/components/app/misc';

 	interface PreviewItem {
@@ -7,6 +7,7 @@
 		name: string;
 		isImage: boolean;
 		isAudio: boolean;
+		isVideo: boolean;
 		preview?: string;
 	}

@@ -49,6 +50,8 @@
 						>
 							{#if item.isAudio}
 								<Music class="h-4 w-4 text-white/70" />
+							{:else if item.isVideo}
+								<Video class="h-4 w-4 text-white/70" />
 							{:else}
 								<FileText class="h-4 w-4 text-white/70" />
 							{/if}
@@ -23,6 +23,7 @@
 		class?: string;
 		disabled?: boolean;
 		hasAudioModality?: boolean;
+		hasVideoModality?: boolean;
 		hasVisionModality?: boolean;
 		hasMcpPromptsSupport?: boolean;
 		hasMcpResourcesSupport?: boolean;
@@ -37,6 +38,7 @@
 		class: className = '',
 		disabled = false,
 		hasAudioModality = false,
+		hasVideoModality = false,
 		hasVisionModality = false,
 		hasMcpPromptsSupport = false,
 		hasMcpResourcesSupport = false,
@@ -58,6 +60,7 @@
 		() => ({
 			hasVisionModality,
 			hasAudioModality,
+			hasVideoModality,
 			hasMcpPromptsSupport,
 			hasMcpResourcesSupport
 		}),
@@ -19,6 +19,7 @@
 		class?: string;
 		disabled?: boolean;
 		hasAudioModality?: boolean;
+		hasVideoModality?: boolean;
 		hasVisionModality?: boolean;
 		hasMcpPromptsSupport?: boolean;
 		hasMcpResourcesSupport?: boolean;
@@ -34,6 +35,7 @@
 		disabled = false,
 		hasAudioModality = false,
 		hasVisionModality = false,
+		hasVideoModality = false,
 		hasMcpPromptsSupport = false,
 		hasMcpResourcesSupport = false,
 		onFileUpload,
@@ -49,6 +51,7 @@
 		() => ({
 			hasVisionModality,
 			hasAudioModality,
+			hasVideoModality,
 			hasMcpPromptsSupport,
 			hasMcpResourcesSupport
 		}),
@@ -7,6 +7,7 @@
 	interface Props {
 		disabled?: boolean;
 		hasAudioModality?: boolean;
+		hasVideoModality?: boolean;
 		hasMcpPromptsSupport?: boolean;
 		hasMcpResourcesSupport?: boolean;
 		hasVisionModality?: boolean;
@@ -20,6 +21,7 @@
 	let {
 		disabled = false,
 		hasAudioModality = false,
+		hasVideoModality = false,
 		hasMcpPromptsSupport = false,
 		hasMcpResourcesSupport = false,
 		hasVisionModality = false,
@@ -37,6 +39,7 @@
 	<ChatFormActionAddSheet
 		{disabled}
 		{hasAudioModality}
+		{hasVideoModality}
 		{hasVisionModality}
 		{hasMcpPromptsSupport}
 		{hasMcpResourcesSupport}
@@ -52,6 +55,7 @@
 	<ChatFormActionAddDropdown
 		{disabled}
 		{hasAudioModality}
+		{hasVideoModality}
 		{hasVisionModality}
 		{hasMcpPromptsSupport}
 		{hasMcpResourcesSupport}
@@ -11,6 +11,7 @@
 		disabled?: boolean;
 		forceForegroundText?: boolean;
 		hasAudioModality?: boolean;
+		hasVideoModality?: boolean;
 		hasVisionModality?: boolean;
 		hasModelSelected?: boolean;
 		isSelectedModelInCache?: boolean;
@@ -23,6 +24,7 @@
 		disabled = false,
 		forceForegroundText = false,
 		hasAudioModality = $bindable(false),
+		hasVideoModality = $bindable(false),
 		hasVisionModality = $bindable(false),
 		hasModelSelected = $bindable(false),
 		isSelectedModelInCache = $bindable(true),
@@ -95,6 +97,10 @@
 		hasAudioModality = activeModelId ? modelsStore.modelSupportsAudio(activeModelId) : false;
 	});

+	$effect(() => {
+		hasVideoModality = activeModelId ? modelsStore.modelSupportsVideo(activeModelId) : false;
+	});
+
 	$effect(() => {
 		void modelPropsVersion;

@@ -66,6 +66,7 @@
 	});

 	let hasAudioModality = $state(false);
+	let hasVideoModality = $state(false);
 	let hasVisionModality = $state(false);
 	let hasModelSelected = $state(false);
 	let isSelectedModelInCache = $state(true);
@@ -94,6 +95,7 @@
 			<ChatFormActionsAdd
 				{disabled}
 				{hasAudioModality}
+				{hasVideoModality}
 				{hasVisionModality}
 				{hasMcpPromptsSupport}
 				{hasMcpResourcesSupport}
@@ -111,6 +113,7 @@
 			{disabled}
 			bind:this={selectorModelRef}
 			bind:hasAudioModality
+			bind:hasVideoModality
 			bind:hasVisionModality
 			bind:hasModelSelected
 			bind:isSelectedModelInCache
@@ -144,6 +144,16 @@
 		return false;
 	});

+	let hasVideoModality = $derived.by(() => {
+		if (activeModelId) {
+			void modelPropsVersion;
+
+			return modelsStore.modelSupportsVideo(activeModelId);
+		}
+
+		return false;
+	});
+
 	let hasVisionModality = $derived.by(() => {
 		if (activeModelId) {
 			void modelPropsVersion;
@@ -284,7 +294,11 @@
 		}

 		// Use model-specific capabilities for file validation
-		const capabilities = { hasVision: hasVisionModality, hasAudio: hasAudioModality };
+		const capabilities = {
+			hasVision: hasVisionModality,
+			hasAudio: hasAudioModality,
+			hasVideo: hasVideoModality
+		};
 		const { supportedFiles, unsupportedFiles, modalityReasons } = filterFilesByModalities(
 			generallySupported,
 			capabilities
@@ -297,6 +311,7 @@

 			if (hasVisionModality) supportedTypes.push('images');
 			if (hasAudioModality) supportedTypes.push('audio files');
+			if (hasVideoModality) supportedTypes.push('video files');

 			fileErrorData = {
 				generallyUnsupported,