nikola/llama_cpp
1
0
Fork 0
Code Issues Pull Requests Actions 19 Packages Projects Releases Wiki Activity

mtmd: mtmd_audio_streaming_istft (#18645)

Browse Source 
Change is decoupled from https://github.com/ggml-org/llama.cpp/pull/18641.

[LFM2.5-Audio-1.5B](https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B)
needs streaming istft for generating output audio.

* add streaming ISTFT class (`mtmd_audio_streaming_istft`) with overlap-add for audio reconstruction
* replace global audio cache with per-instance cache, the model requires
  two independent caches, for preprocessing (audio input) and for istft
  (audio output).
* unified templated FFT/IFFT implementation supporting both forward and inverse transforms
This commit is contained in:
Tarek Dakhran
2026-01-06 21:00:29 +01:00
committed by GitHub
parent 68b4d516c3
commit ccbc84a537
2 changed files with 428 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/mtmd-audio.cpp
+265 -125
View File
@@ -9,32 +9,11 @@
#include <fstream> #include <fstream>
#include <algorithm> #include <algorithm>
// most of the code here is copied from whisper.cpp // some of the code here is copied from whisper.cpp
constexpr bool DEBUG = false; constexpr bool DEBUG = false;
struct mtmd_audio_mel_filters { void mtmd_audio_cache::fill_sin_cos_table(int n) {
int32_t n_mel;
int32_t n_fft;
std::vector<float> data;
};
// note: this global cache is shared among all preprocessors
// if we want to use multiple preprocessors at the same time,
// we will need to enclose it in the preprocessor class in the future
static struct mtmd_audio_global_cache {
// precomputed sin/cos table for FFT
std::vector<float> sin_vals;
std::vector<float> cos_vals;
// hann window
std::vector<float> hann_window;
// mel filter bank
mtmd_audio_mel_filters filters;
void fill_sin_cos_table(int n) {
sin_vals.resize(n); sin_vals.resize(n);
cos_vals.resize(n); cos_vals.resize(n);
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
@@ -42,9 +21,9 @@ static struct mtmd_audio_global_cache {
sin_vals[i] = sinf(theta); sin_vals[i] = sinf(theta);
cos_vals[i] = cosf(theta); cos_vals[i] = cosf(theta);
} }
} }
void fill_hann_window(int length, bool periodic) { void mtmd_audio_cache::fill_hann_window(int length, bool periodic) {
hann_window.resize(length); hann_window.resize(length);
int offset = -1; int offset = -1;
if (periodic) { if (periodic) {
@@ -53,19 +32,15 @@ static struct mtmd_audio_global_cache {
for (int i = 0; i < length; i++) { for (int i = 0; i < length; i++) {
hann_window[i] = 0.5 * (1.0 - cosf((2.0 * M_PI * i) / (length + offset))); hann_window[i] = 0.5 * (1.0 - cosf((2.0 * M_PI * i) / (length + offset)));
} }
} }
// Build mel filterbank matrix [n_mel × n_fft_bins] at runtime. void mtmd_audio_cache::fill_mel_filterbank_matrix(int n_mel,
// n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257.
void fill_mel_filterbank_matrix(
int n_mel,
int n_fft, int n_fft,
int sample_rate, // e.g. 16000 int sample_rate,
float fmin = 0.0f, // e.g. 0.0 float fmin,
float fmax = -1.0f, // e.g. sr/2; pass -1 for auto float fmax,
bool slaney_area_norm = true, bool slaney_area_norm,
float scale = 1.0f // optional extra scaling; use 1.0f/1000.0f to mimic your code float scale) {
) {
GGML_ASSERT(n_mel > 0 && n_fft > 1); GGML_ASSERT(n_mel > 0 && n_fft > 1);
if (fmax <= 0.0f) { if (fmax <= 0.0f) {
fmax = 0.5f * sample_rate; fmax = 0.5f * sample_rate;
@@ -136,80 +111,148 @@ static struct mtmd_audio_global_cache {
} }
} }
} }
} }
} g_cache;
// naive Discrete Fourier Transform // Unified DFT implementation for both forward and inverse transforms
// input is real-valued // Template parameters:
// output is complex-valued // Inverse: false = DFT with exp(-2πi·k·n/N), no scaling
static void dft(const float * in, int N, float * out) { // true = IDFT with exp(+2πi·k·n/N), scales by 1/N
const int n_sin_cos_vals = g_cache.sin_vals.size(); // RealInput: true = input is real-valued (stride 1), avoids imaginary computations
// false = input is complex-valued (interleaved real/imag, stride 2)
template <bool Inverse, bool RealInput>
static void dft_impl(const mtmd_audio_cache & cache, const float * in, int N, float * out) {
const int n_sin_cos_vals = cache.sin_vals.size();
const int sin_cos_step = n_sin_cos_vals / N; const int sin_cos_step = n_sin_cos_vals / N;
constexpr float sign = Inverse ? 1.0f : -1.0f;
const float scale = Inverse ? (1.0f / N) : 1.0f;
for (int k = 0; k < N; k++) { for (int k = 0; k < N; k++) {
float re = 0; float re = 0;
float im = 0; float im = 0;
for (int n = 0; n < N; n++) { for (int n = 0; n < N; n++) {
int idx = (k * n * sin_cos_step) % (n_sin_cos_vals); // t = 2*M_PI*k*n/N int idx = (k * n * sin_cos_step) % n_sin_cos_vals;
re += in[n] * g_cache.cos_vals[idx]; // cos(t) float cos_val = cache.cos_vals[idx];
im -= in[n] * g_cache.sin_vals[idx]; // sin(t) float sin_val = cache.sin_vals[idx];
if constexpr (RealInput) {
// Real input: in_im = 0, simplifies to:
// re += in_re * cos_val
// im += sign * in_re * sin_val
float in_re = in[n];
re += in_re * cos_val;
im += sign * in_re * sin_val;
} else {
float in_re = in[n * 2 + 0];
float in_im = in[n * 2 + 1];
// (a + bi) * (cos + sign*i*sin) = (a*cos - sign*b*sin) + (sign*a*sin + b*cos)i
re += in_re * cos_val - sign * in_im * sin_val;
im += sign * in_re * sin_val + in_im * cos_val;
}
} }
out[k*2 + 0] = re; out[k * 2 + 0] = re * scale;
out[k*2 + 1] = im; out[k * 2 + 1] = im * scale;
} }
} }
// Cooley-Tukey FFT // Cooley-Tukey FFT/IFFT unified implementation
// poor man's implementation - use something better // Template parameters:
// input is real-valued // Inverse: false = FFT with exp(-2πi·k/N), no scaling
// output is complex-valued // true = IFFT with exp(+2πi·k/N), scales by 0.5 at each level
static void fft(float * in, int N, float * out) { // RealInput: true = input is real-valued (stride 1)
const int n_sin_cos_vals = g_cache.sin_vals.size(); // false = input is complex-valued (interleaved real/imag, stride 2)
template <bool Inverse, bool RealInput>
static void fft_impl(const mtmd_audio_cache & cache, float * in, int N, float * out) {
const int n_sin_cos_vals = cache.sin_vals.size();
if (N == 1) { if (N == 1) {
out[0] = in[0]; out[0] = in[0];
out[1] = 0; if constexpr (RealInput) {
out[1] = 0.0f;
} else {
out[1] = in[1];
}
return; return;
} }
const int half_N = N / 2; const int half_N = N / 2;
if (N - half_N*2 == 1) { if (N - half_N * 2 == 1) {
dft(in, N, out); // Odd N: fall back to DFT
dft_impl<Inverse, RealInput>(cache, in, N, out);
return; return;
} }
float* even = in + N; // Split into even and odd
if constexpr (RealInput) {
// Real input: stride is 1, copy only real values
float * even = in + N;
for (int i = 0; i < half_N; ++i) { for (int i = 0; i < half_N; ++i) {
even[i]= in[2*i]; even[i] = in[2 * i];
} }
float* even_fft = out + 2 * N; float * even_fft = out + 2 * N;
fft(even, half_N, even_fft); fft_impl<Inverse, true>(cache, even, half_N, even_fft);
float* odd = even; float * odd = even;
for (int i = 0; i < half_N; ++i) { for (int i = 0; i < half_N; ++i) {
odd[i] = in[2*i + 1]; odd[i] = in[2 * i + 1];
} }
float* odd_fft = even_fft + N; float * odd_fft = even_fft + N;
fft(odd, half_N, odd_fft); fft_impl<Inverse, true>(cache, odd, half_N, odd_fft);
} else {
// Complex input: stride is 2, copy complex pairs
float * even = in + N * 2;
for (int i = 0; i < half_N; ++i) {
even[i * 2 + 0] = in[2 * i * 2 + 0];
even[i * 2 + 1] = in[2 * i * 2 + 1];
}
float * even_fft = out + 2 * N;
fft_impl<Inverse, false>(cache, even, half_N, even_fft);
float * odd = even;
for (int i = 0; i < half_N; ++i) {
odd[i * 2 + 0] = in[(2 * i + 1) * 2 + 0];
odd[i * 2 + 1] = in[(2 * i + 1) * 2 + 1];
}
float * odd_fft = even_fft + N;
fft_impl<Inverse, false>(cache, odd, half_N, odd_fft);
}
float * even_fft = out + 2 * N;
float * odd_fft = even_fft + N;
const int sin_cos_step = n_sin_cos_vals / N; const int sin_cos_step = n_sin_cos_vals / N;
constexpr float sign = Inverse ? 1.0f : -1.0f;
constexpr float scale = Inverse ? 0.5f : 1.0f;
for (int k = 0; k < half_N; k++) { for (int k = 0; k < half_N; k++) {
int idx = k * sin_cos_step; // t = 2*M_PI*k/N int idx = k * sin_cos_step; // t = 2*M_PI*k/N
float re = g_cache.cos_vals[idx]; // cos(t) float re = cache.cos_vals[idx];
float im = -g_cache.sin_vals[idx]; // sin(t) float im = sign * cache.sin_vals[idx];
float re_odd = odd_fft[2*k + 0]; float re_odd = odd_fft[2 * k + 0];
float im_odd = odd_fft[2*k + 1]; float im_odd = odd_fft[2 * k + 1];
out[2*k + 0] = even_fft[2*k + 0] + re*re_odd - im*im_odd; out[2 * k + 0] = scale * (even_fft[2 * k + 0] + re * re_odd - im * im_odd);
out[2*k + 1] = even_fft[2*k + 1] + re*im_odd + im*re_odd; out[2 * k + 1] = scale * (even_fft[2 * k + 1] + re * im_odd + im * re_odd);
out[2*(k + half_N) + 0] = even_fft[2*k + 0] - re*re_odd + im*im_odd; out[2 * (k + half_N) + 0] = scale * (even_fft[2 * k + 0] - re * re_odd + im * im_odd);
out[2*(k + half_N) + 1] = even_fft[2*k + 1] - re*im_odd - im*re_odd; out[2 * (k + half_N) + 1] = scale * (even_fft[2 * k + 1] - re * im_odd - im * re_odd);
} }
} }
// Forward FFT for real input (used by mel spectrogram)
static void fft(const mtmd_audio_cache & cache, float * in, int N, float * out) {
fft_impl<false, true>(cache, in, N, out);
}
// Inverse FFT for complex input
static void ifft(const mtmd_audio_cache & cache, float * in, int N, float * out) {
fft_impl<true, false>(cache, in, N, out);
}
struct filter_params { struct filter_params {
int32_t n_mel; int32_t n_mel;
int32_t n_fft_bins; int32_t n_fft_bins;
@@ -222,20 +265,27 @@ struct filter_params {
bool norm_per_feature = false; bool norm_per_feature = false;
}; };
static void log_mel_spectrogram_worker_thread(int ith, const float * hann, const std::vector<float> & samples, static void log_mel_spectrogram_worker_thread(int ith,
int n_samples, int frame_size, int frame_step, int n_threads, const float * hann,
const filter_params & params, mtmd_audio_mel & out) { const std::vector<float> & samples,
int n_samples,
int frame_size,
int frame_step,
int n_threads,
const filter_params & params,
const mtmd_audio_cache & cache,
mtmd_audio_mel & out) {
std::vector<float> fft_in(frame_size * 2, 0.0); std::vector<float> fft_in(frame_size * 2, 0.0);
std::vector<float> fft_out(frame_size * 2 * 2 * 2); std::vector<float> fft_out(frame_size * 2 * 2 * 2);
int n_fft_bins = params.n_fft_bins; int n_fft_bins = params.n_fft_bins;
int i = ith; int i = ith;
const auto & filters = g_cache.filters; const auto & filters = cache.filters;
// make sure n_fft == 1 + (WHISPER_N_FFT / 2), bin_0 to bin_nyquist // make sure n_fft == 1 + (WHISPER_N_FFT / 2), bin_0 to bin_nyquist
GGML_ASSERT(n_fft_bins == 1 + (frame_size / 2)); GGML_ASSERT(n_fft_bins == 1 + (frame_size / 2));
GGML_ASSERT(g_cache.sin_vals.size() == g_cache.cos_vals.size()); GGML_ASSERT(cache.sin_vals.size() == cache.cos_vals.size());
// calculate FFT only when fft_in are not all zero // calculate FFT only when fft_in are not all zero
for (; i < std::min(n_samples / frame_step + 1, out.n_len); i += n_threads) { for (; i < std::min(n_samples / frame_step + 1, out.n_len); i += n_threads) {
const int offset = i * frame_step; const int offset = i * frame_step;
@@ -251,7 +301,7 @@ static void log_mel_spectrogram_worker_thread(int ith, const float * hann, const
} }
// FFT // FFT
fft(fft_in.data(), frame_size, fft_out.data()); fft(cache, fft_in.data(), frame_size, fft_out.data());
// Calculate modulus^2 of complex numbers // Calculate modulus^2 of complex numbers
// Use pow(fft_out[2 * j + 0], 2) + pow(fft_out[2 * j + 1], 2) causes inference quality problem? Interesting. // Use pow(fft_out[2 * j + 0], 2) + pow(fft_out[2 * j + 1], 2) causes inference quality problem? Interesting.
@@ -298,6 +348,7 @@ static bool log_mel_spectrogram(
const int n_samples_in, const int n_samples_in,
const int n_threads, const int n_threads,
const filter_params & params, const filter_params & params,
const mtmd_audio_cache & cache,
mtmd_audio_mel & out) { mtmd_audio_mel & out) {
//const int64_t t_start_us = ggml_time_us(); //const int64_t t_start_us = ggml_time_us();
@@ -305,7 +356,7 @@ static bool log_mel_spectrogram(
int n_samples = n_samples_in; int n_samples = n_samples_in;
// Hann window // Hann window
const float * hann = g_cache.hann_window.data(); const float * hann = cache.hann_window.data();
const int frame_size = (params.n_fft_bins - 1) * 2; const int frame_size = (params.n_fft_bins - 1) * 2;
const int frame_step = params.hop_length; const int frame_step = params.hop_length;
@@ -372,14 +423,14 @@ static bool log_mel_spectrogram(
{ {
std::vector<std::thread> workers(n_threads - 1); std::vector<std::thread> workers(n_threads - 1);
for (int iw = 0; iw < n_threads - 1; ++iw) { for (int iw = 0; iw < n_threads - 1; ++iw) {
workers[iw] = std::thread( workers[iw] =
log_mel_spectrogram_worker_thread, iw + 1, hann, std::cref(samples_padded), std::thread(log_mel_spectrogram_worker_thread, iw + 1, hann, std::cref(samples_padded), n_samples,
n_samples, frame_size, frame_step, n_threads, frame_size, frame_step, n_threads, std::cref(params), std::cref(cache), std::ref(out));
std::cref(params), std::ref(out));
} }
// main thread // main thread
log_mel_spectrogram_worker_thread(0, hann, samples_padded, n_samples, frame_size, frame_step, n_threads, params, out); log_mel_spectrogram_worker_thread(0, hann, samples_padded, n_samples, frame_size, frame_step, n_threads, params,
cache, out);
for (int iw = 0; iw < n_threads - 1; ++iw) { for (int iw = 0; iw < n_threads - 1; ++iw) {
workers[iw].join(); workers[iw].join();
} }
@@ -450,16 +501,12 @@ static bool log_mel_spectrogram(
// //
void mtmd_audio_preprocessor_whisper::initialize() { void mtmd_audio_preprocessor_whisper::initialize() {
g_cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_sin_cos_table(hparams.audio_n_fft);
g_cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_hann_window(hparams.audio_window_len, true);
g_cache.fill_mel_filterbank_matrix( cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
hparams.n_mel_bins,
hparams.audio_n_fft,
hparams.audio_sample_rate);
} }
bool mtmd_audio_preprocessor_whisper::preprocess( bool mtmd_audio_preprocessor_whisper::preprocess(const float * samples,
const float * samples,
size_t n_samples, size_t n_samples,
std::vector<mtmd_audio_mel> & output) { std::vector<mtmd_audio_mel> & output) {
if (n_samples == 0) { if (n_samples == 0) {
@@ -471,7 +518,7 @@ bool mtmd_audio_preprocessor_whisper::preprocess(
// if input is too short, pad with zeros // if input is too short, pad with zeros
// this is to avoid potential issues with stage1/2 padding in log_mel_spectrogram // this is to avoid potential issues with stage1/2 padding in log_mel_spectrogram
// TODO: maybe handle this better // TODO: maybe handle this better
size_t min_samples = (size_t)hparams.audio_sample_rate * (hparams.audio_chunk_len + 1); // +1 second margin size_t min_samples = (size_t) hparams.audio_sample_rate * (hparams.audio_chunk_len + 1); // +1 second margin
if (n_samples < min_samples) { if (n_samples < min_samples) {
smpl.resize(min_samples, 0.0f); smpl.resize(min_samples, 0.0f);
std::memcpy(smpl.data(), samples, n_samples * sizeof(float)); std::memcpy(smpl.data(), samples, n_samples * sizeof(float));
@@ -490,18 +537,15 @@ bool mtmd_audio_preprocessor_whisper::preprocess(
params.use_natural_log = false; params.use_natural_log = false;
params.norm_per_feature = false; params.norm_per_feature = false;
// make sure the global cache is initialized // make sure the cache is initialized
GGML_ASSERT(!g_cache.sin_vals.empty()); GGML_ASSERT(!cache.sin_vals.empty());
GGML_ASSERT(!g_cache.cos_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty());
GGML_ASSERT(!g_cache.filters.data.empty()); GGML_ASSERT(!cache.filters.data.empty());
mtmd_audio_mel out_full; mtmd_audio_mel out_full;
bool ok = log_mel_spectrogram( bool ok = log_mel_spectrogram(samples, n_samples,
samples,
n_samples,
4, // n_threads 4, // n_threads
params, params, cache, out_full);
out_full);
if (!ok) { if (!ok) {
return false; return false;
} }
@@ -512,10 +556,10 @@ bool mtmd_audio_preprocessor_whisper::preprocess(
printf("output: n_mel = %d, n_len = %d\n", out_full.n_mel, out_full.n_len); printf("output: n_mel = %d, n_len = %d\n", out_full.n_mel, out_full.n_len);
} }
const size_t frames_per_chunk = 3000; const size_t frames_per_chunk = 3000;
GGML_ASSERT((size_t)out_full.n_len > frames_per_chunk); GGML_ASSERT((size_t) out_full.n_len > frames_per_chunk);
for (size_t off = 0; off < (size_t)out_full.n_len; off += frames_per_chunk) { for (size_t off = 0; off < (size_t) out_full.n_len; off += frames_per_chunk) {
int n_len = std::min(frames_per_chunk, (size_t)out_full.n_len - off); int n_len = std::min(frames_per_chunk, (size_t) out_full.n_len - off);
if ((size_t)n_len < frames_per_chunk) { if ((size_t) n_len < frames_per_chunk) {
break; // last uncomplete chunk will always be a padded chunk, safe to ignore break; // last uncomplete chunk will always be a padded chunk, safe to ignore
} }
@@ -526,7 +570,7 @@ bool mtmd_audio_preprocessor_whisper::preprocess(
out_chunk.data.reserve(out_chunk.n_mel * out_chunk.n_len); out_chunk.data.reserve(out_chunk.n_mel * out_chunk.n_len);
for (int i = 0; i < out_full.n_mel; i++) { for (int i = 0; i < out_full.n_mel; i++) {
auto src = out_full.data.begin() + i*out_full.n_len + off; auto src = out_full.data.begin() + i * out_full.n_len + off;
out_chunk.data.insert(out_chunk.data.end(), src, src + frames_per_chunk); out_chunk.data.insert(out_chunk.data.end(), src, src + frames_per_chunk);
} }
@@ -541,16 +585,12 @@ bool mtmd_audio_preprocessor_whisper::preprocess(
// //
void mtmd_audio_preprocessor_conformer::initialize() { void mtmd_audio_preprocessor_conformer::initialize() {
g_cache.fill_sin_cos_table(hparams.audio_n_fft); cache.fill_sin_cos_table(hparams.audio_n_fft);
g_cache.fill_hann_window(hparams.audio_window_len, true); cache.fill_hann_window(hparams.audio_window_len, true);
g_cache.fill_mel_filterbank_matrix( cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
hparams.n_mel_bins,
hparams.audio_n_fft,
hparams.audio_sample_rate);
} }
bool mtmd_audio_preprocessor_conformer::preprocess( bool mtmd_audio_preprocessor_conformer::preprocess(const float * samples,
const float * samples,
size_t n_samples, size_t n_samples,
std::vector<mtmd_audio_mel> & output) { std::vector<mtmd_audio_mel> & output) {
// empty audio // empty audio
@@ -569,18 +609,15 @@ bool mtmd_audio_preprocessor_conformer::preprocess(
params.use_natural_log = true; params.use_natural_log = true;
params.norm_per_feature = true; params.norm_per_feature = true;
// make sure the global cache is initialized // make sure the cache is initialized
GGML_ASSERT(!g_cache.sin_vals.empty()); GGML_ASSERT(!cache.sin_vals.empty());
GGML_ASSERT(!g_cache.cos_vals.empty()); GGML_ASSERT(!cache.cos_vals.empty());
GGML_ASSERT(!g_cache.filters.data.empty()); GGML_ASSERT(!cache.filters.data.empty());
mtmd_audio_mel out_full; mtmd_audio_mel out_full;
bool ok = log_mel_spectrogram( bool ok = log_mel_spectrogram(samples, n_samples,
samples,
n_samples,
4, // n_threads 4, // n_threads
params, params, cache, out_full);
out_full);
if (!ok) { if (!ok) {
return false; return false;
} }
@@ -588,3 +625,106 @@ bool mtmd_audio_preprocessor_conformer::preprocess(
output.push_back(std::move(out_full)); output.push_back(std::move(out_full));
return true; return true;
} }
//
// mtmd_audio_streaming_istft implementation
//
mtmd_audio_streaming_istft::mtmd_audio_streaming_istft(int n_fft, int hop_length) :
n_fft(n_fft),
hop_length(hop_length),
n_fft_bins(n_fft / 2 + 1),
overlap_buffer(n_fft, 0.0f),
window_sum_buffer(n_fft, 0.0f),
padding_to_remove((n_fft - hop_length) / 2),
ifft_in(n_fft * 2 * 4, 0.0f), // extra space for recursive IFFT
ifft_out(n_fft * 2 * 4, 0.0f) {
cache.fill_sin_cos_table(n_fft);
cache.fill_hann_window(n_fft, true);
}
void mtmd_audio_streaming_istft::reset() {
std::fill(overlap_buffer.begin(), overlap_buffer.end(), 0.0f);
std::fill(window_sum_buffer.begin(), window_sum_buffer.end(), 0.0f);
padding_to_remove = (n_fft - hop_length) / 2;
}
std::vector<float> mtmd_audio_streaming_istft::process_frame(const float * frame_spectrum) {
std::vector<float> output(hop_length);
// copy frequencies
for (int j = 0; j < n_fft_bins; j++) {
ifft_in[j * 2 + 0] = frame_spectrum[j * 2 + 0];
ifft_in[j * 2 + 1] = frame_spectrum[j * 2 + 1];
}
// mirror negative frequencies
for (int j = 1; j < n_fft_bins - 1; j++) {
int mirror_idx = n_fft - j;
ifft_in[mirror_idx * 2 + 0] = ifft_in[j * 2 + 0];
ifft_in[mirror_idx * 2 + 1] = -ifft_in[j * 2 + 1]; // conjugate
}
ifft(cache, ifft_in.data(), n_fft, ifft_out.data());
// update window sum and overlap buffer
for (int j = 0; j < n_fft; j++) {
window_sum_buffer[j] += cache.hann_window[j] * cache.hann_window[j];
overlap_buffer[j] += ifft_out[j * 2] * cache.hann_window[j];
}
// extract hop_length samples with normalization
for (int i = 0; i < hop_length; i++) {
if (window_sum_buffer[i] > 1e-8f) {
output[i] = overlap_buffer[i] / window_sum_buffer[i];
} else {
output[i] = overlap_buffer[i];
}
}
// shift buffers left by hop_length
std::copy(overlap_buffer.begin() + hop_length, overlap_buffer.end(), overlap_buffer.begin());
std::fill(overlap_buffer.end() - hop_length, overlap_buffer.end(), 0.0f);
std::copy(window_sum_buffer.begin() + hop_length, window_sum_buffer.end(), window_sum_buffer.begin());
std::fill(window_sum_buffer.end() - hop_length, window_sum_buffer.end(), 0.0f);
// Remove padding if needed
int to_remove = std::min(padding_to_remove, (int) output.size());
padding_to_remove -= to_remove;
output.erase(output.begin(), output.begin() + to_remove);
return output;
}
std::vector<float> mtmd_audio_streaming_istft::flush() {
std::vector<float> output;
// Extract remaining samples from overlap buffer
// Continue until we've extracted all meaningful samples
int remaining = n_fft - hop_length;
while (remaining > 0) {
int chunk_size = std::min(remaining, hop_length);
for (int i = 0; i < chunk_size; i++) {
float sample;
if (window_sum_buffer[i] > 1e-8f) {
sample = overlap_buffer[i] / window_sum_buffer[i];
} else {
sample = overlap_buffer[i];
}
output.push_back(sample);
}
// Shift buffers
std::copy(overlap_buffer.begin() + chunk_size, overlap_buffer.end(), overlap_buffer.begin());
std::fill(overlap_buffer.end() - chunk_size, overlap_buffer.end(), 0.0f);
std::copy(window_sum_buffer.begin() + chunk_size, window_sum_buffer.end(), window_sum_buffer.begin());
std::fill(window_sum_buffer.end() - chunk_size, window_sum_buffer.end(), 0.0f);
remaining -= chunk_size;
}
return output;
}
tools/mtmd/mtmd-audio.h
+73
View File
@@ -17,6 +17,38 @@ struct mtmd_audio_mel {
std::vector<float> data; std::vector<float> data;
}; };
struct mtmd_audio_mel_filters {
int32_t n_mel;
int32_t n_fft;
std::vector<float> data;
};
// cache for audio processing, each processor instance owns its own cache
struct mtmd_audio_cache {
std::vector<float> sin_vals;
std::vector<float> cos_vals;
std::vector<float> hann_window;
mtmd_audio_mel_filters filters;
void fill_sin_cos_table(int n);
void fill_hann_window(int length, bool periodic);
// Build mel filterbank matrix [n_mel × n_fft_bins] at runtime.
// n_fft_bins must be (N_fft / 2 + 1). Example: if N_fft=512 -> n_fft_bins=257.
void fill_mel_filterbank_matrix(int n_mel,
int n_fft,
int sample_rate, // e.g. 16000
float fmin = 0.0f, // e.g. 0.0
float fmax = -1.0f, // e.g. sr/2; pass -1 for auto
bool slaney_area_norm = true,
float scale = 1.0f // optional extra scaling
);
};
struct mtmd_audio_preprocessor { struct mtmd_audio_preprocessor {
const clip_hparams & hparams; const clip_hparams & hparams;
@@ -31,10 +63,51 @@ struct mtmd_audio_preprocessor_whisper : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} mtmd_audio_preprocessor_whisper(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override; void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override; bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
private:
mtmd_audio_cache cache;
}; };
struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor { struct mtmd_audio_preprocessor_conformer : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} mtmd_audio_preprocessor_conformer(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override; void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override; bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
private:
mtmd_audio_cache cache;
};
//
// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
//
struct mtmd_audio_streaming_istft {
mtmd_audio_streaming_istft(int n_fft, int hop_length);
// reset streaming state
void reset();
// process a single STFT frame (streaming)
// frame_spectrum: [n_fft_bins x 2] interleaved real/imag
// returns: up to hop_length samples
std::vector<float> process_frame(const float * frame_spectrum);
// flush remaining samples at end of stream
std::vector<float> flush();
private:
int n_fft;
int hop_length;
int n_fft_bins;
// Own cache for output processing
mtmd_audio_cache cache;
// Streaming state
std::vector<float> overlap_buffer;
std::vector<float> window_sum_buffer;
int padding_to_remove;
// Working buffers for IFFT
std::vector<float> ifft_in;
std::vector<float> ifft_out;
}; };