UI/jpeg exif orientation (#24196)
* ui: bake jpeg exif orientation into uploaded images stb_image in mtmd ignores exif metadata, so rotated smartphone photos reach the model with raw pixel orientation. The webui now reads the exif orientation tag at send time and feeds it into the existing capImageDataURLSize canvas pass: the browser applies the rotation when decoding, so capped images come out upright for free, and images under the cap threshold get a single plain redraw when orientation > 1. At most one re-encode ever happens per image. Upright jpegs with capping disabled pass through untouched, bit perfect. Adds jpeg-orientation.ts with a minimal exif parser working on a bounded base64 prefix (both endianness, returns 1 on any malformed input) and unit tests against handcrafted jpeg byte streams. * ui: move jpeg exif constants into lib/constants * ui: add browser test for jpeg orientation and capping Covers capImageDataURLSize end to end in chromium with real Pillow generated jpeg fixtures across exif orientations 1/3/5/6/8: upright quadrant colors checked pixel-wise, expected dimensions with and without capping, no orientation tag left in the output, and strict passthrough when nothing needs rewriting.
This commit is contained in:
Pascal
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* JPEG and EXIF binary format constants for orientation parsing.
|
||||
*/
|
||||
|
||||
/** Bytes of file prefix to scan, the APP1 EXIF segment sits near the start */
|
||||
export const EXIF_SCAN_BYTE_LIMIT = 128 * 1024;
|
||||
|
||||
/** JPEG start of image marker */
|
||||
export const JPEG_SOI_MARKER = 0xffd8;
|
||||
|
||||
/** APP1 segment marker byte, carries the EXIF payload */
|
||||
export const APP1_MARKER = 0xe1;
|
||||
|
||||
/** Start of scan marker byte, compressed data begins and no EXIF follows */
|
||||
export const SOS_MARKER = 0xda;
|
||||
|
||||
/** "Exif" signature opening the APP1 payload, big endian uint32 */
|
||||
export const EXIF_SIGNATURE = 0x45786966;
|
||||
|
||||
/** TIFF byte order mark for little endian ("II") */
|
||||
export const TIFF_LITTLE_ENDIAN = 0x4949;
|
||||
|
||||
/** TIFF magic number following the byte order mark */
|
||||
export const TIFF_MAGIC = 42;
|
||||
|
||||
/** EXIF tag id holding the orientation value */
|
||||
export const EXIF_ORIENTATION_TAG = 0x0112;
|
||||
|
||||
/** Size in bytes of one IFD directory entry */
|
||||
export const IFD_ENTRY_SIZE = 12;
|
||||
@@ -34,7 +34,6 @@ import type {
|
||||
import { modelsStore } from '$lib/stores/models.svelte';
|
||||
import { settingsStore } from '../stores/settings.svelte';
|
||||
import { capImageDataURLSize } from '../utils/cap-img-size';
|
||||
import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size';
|
||||
|
||||
function getAudioInputFormat(mimeType: string): AudioInputFormat {
|
||||
const normalizedMimeType = mimeType.trim().toLowerCase();
|
||||
@@ -961,10 +960,11 @@ export class ChatService {
|
||||
|
||||
for (const image of imageFiles) {
|
||||
const maxImageResolution = settingsStore.getConfig(SETTINGS_KEYS.MAX_IMAGE_RESOLUTION);
|
||||
let base64Url = image.base64Url;
|
||||
if (maxImageResolution > 1 / MEGAPIXELS_TO_PIXELS) {
|
||||
base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution);
|
||||
}
|
||||
|
||||
// Caps the resolution and bakes the jpeg exif orientation in one pass,
|
||||
// untouched images pass through as is
|
||||
const base64Url = await capImageDataURLSize(image.base64Url, maxImageResolution);
|
||||
|
||||
contentParts.push({
|
||||
type: ContentPartType.IMAGE_URL,
|
||||
image_url: { url: base64Url }
|
||||
|
||||
@@ -1,11 +1,19 @@
|
||||
import { MEGAPIXELS_TO_PIXELS } from '$lib/constants/image-size';
|
||||
import { BASE64_IMAGE_URI_REGEX } from '$lib/constants/uri-template';
|
||||
import { getJpegOrientationFromDataURL, isJpegMimeType } from './jpeg-orientation';
|
||||
import { MimeTypeImage } from '$lib/enums';
|
||||
|
||||
/**
|
||||
* Converts an Image base64 data URL to another Image data URL with capped dimensions to reduce file size.
|
||||
*
|
||||
* For JPEGs the EXIF orientation is baked into the pixels in the same canvas
|
||||
* pass, the browser applies the rotation when decoding so naturalWidth and
|
||||
* naturalHeight already describe the upright image. Backends decoding with
|
||||
* stb_image ignore EXIF, see ggml-org/llama.cpp#20870. Images that need
|
||||
* neither capping nor rotation pass through untouched, so at most one
|
||||
* re-encode ever happens.
|
||||
* @param base64UrlImage - The Image base64 data URL to convert
|
||||
* @param maxMegapixels - The maximum image size in megapixels for the output Image
|
||||
* @param maxMegapixels - The maximum image size in megapixels for the output Image, 0 disables capping
|
||||
* @returns Promise resolving to Image data URL
|
||||
*/
|
||||
export function capImageDataURLSize(
|
||||
@@ -26,6 +34,10 @@ export function capImageDataURLSize(
|
||||
return reject(new Error(`Unsupported image MIME type: ${mimeType}`));
|
||||
}
|
||||
|
||||
const orientation = isJpegMimeType(mimeType)
|
||||
? getJpegOrientationFromDataURL(base64UrlImage)
|
||||
: 1;
|
||||
|
||||
const img = new Image();
|
||||
|
||||
img.onload = () => {
|
||||
@@ -46,6 +58,10 @@ export function capImageDataURLSize(
|
||||
const scaleFactor = Math.sqrt(maxPixels / totalPixels);
|
||||
canvas.width = Math.floor(targetWidth * scaleFactor);
|
||||
canvas.height = Math.floor(targetHeight * scaleFactor);
|
||||
} else if (orientation > 1) {
|
||||
// No capping needed but the pixels still need the rotation baked in
|
||||
canvas.width = targetWidth;
|
||||
canvas.height = targetHeight;
|
||||
} else {
|
||||
return resolve(base64UrlImage);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
import {
|
||||
EXIF_SCAN_BYTE_LIMIT,
|
||||
JPEG_SOI_MARKER,
|
||||
APP1_MARKER,
|
||||
SOS_MARKER,
|
||||
EXIF_SIGNATURE,
|
||||
TIFF_LITTLE_ENDIAN,
|
||||
TIFF_MAGIC,
|
||||
EXIF_ORIENTATION_TAG,
|
||||
IFD_ENTRY_SIZE
|
||||
} from '$lib/constants/jpeg-exif';
|
||||
import { MimeTypeImage } from '$lib/enums';
|
||||
|
||||
/**
|
||||
* Read the EXIF orientation tag from a JPEG base64 data URL
|
||||
*
|
||||
* Only a bounded prefix of the base64 payload is decoded, the APP1 segment
|
||||
* always sits near the start of the file.
|
||||
* @param base64UrlJpeg - The JPEG base64 data URL to inspect
|
||||
* @returns The orientation value (1 to 8), or 1 when absent or unreadable
|
||||
*/
|
||||
export function getJpegOrientationFromDataURL(base64UrlJpeg: string): number {
|
||||
try {
|
||||
const payloadStart = base64UrlJpeg.indexOf(',') + 1;
|
||||
|
||||
if (payloadStart <= 0) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Keep the slice a multiple of 4 characters so atob accepts it
|
||||
const charLimit = Math.ceil(EXIF_SCAN_BYTE_LIMIT / 3) * 4;
|
||||
const slice = base64UrlJpeg.slice(payloadStart, payloadStart + charLimit);
|
||||
const binary = atob(slice.slice(0, slice.length - (slice.length % 4)));
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
bytes[i] = binary.charCodeAt(i);
|
||||
}
|
||||
|
||||
return findExifOrientation(new DataView(bytes.buffer));
|
||||
} catch {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the JPEG segments of a header buffer looking for the APP1 EXIF block
|
||||
* @param view - DataView over the JPEG header bytes
|
||||
* @returns The orientation value (1 to 8), or 1 when absent or malformed
|
||||
*/
|
||||
function findExifOrientation(view: DataView): number {
|
||||
if (view.byteLength < 4 || view.getUint16(0) !== JPEG_SOI_MARKER) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
let offset = 2;
|
||||
|
||||
while (offset + 4 <= view.byteLength) {
|
||||
if (view.getUint8(offset) !== 0xff) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const marker = view.getUint8(offset + 1);
|
||||
|
||||
// Compressed image data starts here: no EXIF past this point
|
||||
if (marker === SOS_MARKER) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const segmentLength = view.getUint16(offset + 2);
|
||||
|
||||
if (marker === APP1_MARKER) {
|
||||
return parseExifOrientation(view, offset + 4, segmentLength);
|
||||
}
|
||||
|
||||
offset += 2 + segmentLength;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the orientation tag from an APP1 EXIF payload
|
||||
* @param view - DataView over the JPEG header bytes
|
||||
* @param start - Offset of the APP1 payload, right after the segment length
|
||||
* @param segmentLength - Declared APP1 segment length
|
||||
* @returns The orientation value (1 to 8), or 1 when absent or malformed
|
||||
*/
|
||||
function parseExifOrientation(view: DataView, start: number, segmentLength: number): number {
|
||||
const end = Math.min(start + segmentLength, view.byteLength);
|
||||
|
||||
// The payload opens with the "Exif\0\0" signature
|
||||
if (
|
||||
start + 6 > end ||
|
||||
view.getUint32(start) !== EXIF_SIGNATURE ||
|
||||
view.getUint16(start + 4) !== 0
|
||||
) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const tiff = start + 6;
|
||||
|
||||
if (tiff + 8 > end) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const littleEndian = view.getUint16(tiff) === TIFF_LITTLE_ENDIAN;
|
||||
|
||||
if (view.getUint16(tiff + 2, littleEndian) !== TIFF_MAGIC) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const ifdOffset = view.getUint32(tiff + 4, littleEndian);
|
||||
|
||||
if (tiff + ifdOffset + 2 > end) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
const entryCount = view.getUint16(tiff + ifdOffset, littleEndian);
|
||||
|
||||
// Scan IFD0 entries for the orientation tag
|
||||
for (let i = 0; i < entryCount; i++) {
|
||||
const entry = tiff + ifdOffset + 2 + i * IFD_ENTRY_SIZE;
|
||||
|
||||
if (entry + IFD_ENTRY_SIZE > end) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (view.getUint16(entry, littleEndian) === EXIF_ORIENTATION_TAG) {
|
||||
const orientation = view.getUint16(entry + 8, littleEndian);
|
||||
|
||||
return orientation >= 1 && orientation <= 8 ? orientation : 1;
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a MIME type represents a JPEG
|
||||
* @param mimeType - The MIME type to check
|
||||
* @returns True if the MIME type is a JPEG variant
|
||||
*/
|
||||
export function isJpegMimeType(mimeType: string): boolean {
|
||||
return mimeType === MimeTypeImage.JPEG || mimeType === MimeTypeImage.JPG;
|
||||
}
|
||||
Reference in New Issue
Block a user