mtmd: add chunks and fix preproc for qwen3a (#23073)

* mtmd: add chunks and fix preproc for qwen3a

* add attn_mask

* limit mtmd_chunk size (avoid blow up memory)

* correct audio tokens

* re-order the set_input case

* remove attn_mask
This commit is contained in:
Xuan-Son Nguyen
2026-05-15 19:32:47 +02:00
committed by GitHub
parent 8be1786707
commit 72e60f500d
7 changed files with 200 additions and 71 deletions
+6 -1
View File
@@ -11,6 +11,10 @@
#define DEFAULT_INTERPOLATION_MODE (GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS)
struct build_vit_opts {
ggml_tensor * attn_mask = nullptr;
};
struct clip_graph {
const clip_model & model;
const clip_hparams & hparams;
@@ -63,7 +67,8 @@ struct clip_graph {
norm_type norm_t,
ffn_op_type ffn_t,
ggml_tensor * learned_pos_embd,
std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos);
std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos,
const build_vit_opts & opts = {});
// build the input after conv2d (inp_raw --> patches)
// returns tensor with shape [n_embd, n_patches]