mtmd: support "frame merge" for qwen-vl-based models (#21858)

* feat: add video support for Qwen3.5

* various clean up

* revise the design

* fix llava-uhd case

* nits

* nits 2

---------

Co-authored-by: andrewmd5 <1297077+andrewmd5@users.noreply.github.com>
This commit is contained in:
Xuan-Son Nguyen
2026-06-06 21:17:25 +02:00
committed by GitHub
co-authored by andrewmd5
parent 6b80c74f28
commit 31e82494c0
10 changed files with 197 additions and 74 deletions
+8
View File
@@ -20,6 +20,12 @@ struct clip_image_size {
bool operator==(const clip_image_size & other) const {
return width == other.width && height == other.height;
}
bool operator!=(const clip_image_size & other) const {
return !(*this == other);
}
int area() const {
return width * height;
}
};
struct clip_image_f32;
@@ -101,6 +107,8 @@ bool clip_is_llava(const struct clip_ctx * ctx);
bool clip_has_vision_encoder(const struct clip_ctx * ctx);
bool clip_has_audio_encoder(const struct clip_ctx * ctx);
int clip_model_n_batch_max(const struct clip_ctx * ctx);
std::map<ggml_backend_dev_t, size_t> clip_get_mem_usage(const struct clip_ctx * ctx);
struct clip_cap {