mtmd, model: allow skip build_vit() (#24077)

* add model

* nits
This commit is contained in:
Xuan-Son Nguyen
2026-06-03 17:10:35 +02:00
committed by GitHub
parent ee4cf705bb
commit a731805ced
20 changed files with 363 additions and 3 deletions
+38
View File
@@ -942,6 +942,44 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s
return true;
}
//
// mtmd_audio_preprocessor_gemma4ua
//
void mtmd_audio_preprocessor_gemma4ua::initialize() {
// no-op: no FFT or filterbank needed
}
bool mtmd_audio_preprocessor_gemma4ua::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
if (n_samples == 0) {
return false;
}
const int frame_size = hparams.n_mel_bins; // 640 samples per token @ 16 kHz = 40 ms
const int n_tokens = ((int)n_samples + frame_size - 1) / frame_size;
mtmd_audio_mel mel;
mel.n_len = n_tokens;
mel.n_len_org = n_tokens;
mel.n_mel = frame_size;
mel.data.assign((size_t)frame_size * n_tokens, 0.0f);
// Store mel-major (data[f * n_tokens + t]) so the ggml tensor loads as
// [n_tokens, frame_size] with ne[0]=n_tokens, ne[1]=frame_size.
// The graph builder transposes before RMSNorm so normalization is over frame_size.
for (int t = 0; t < n_tokens; t++) {
for (int f = 0; f < frame_size; f++) {
size_t src = (size_t)t * frame_size + f;
mel.data[(size_t)f * n_tokens + t] = (src < n_samples) ? samples[src] : 0.0f;
}
}
output.push_back(std::move(mel));
return true;
}
//
// mtmd_audio_streaming_istft implementation
//