diff --git a/CLAUDE.md b/CLAUDE.md index 8711bca..3aa7281 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -249,6 +249,18 @@ name, a dotfile, `account.toml`, outside the account's `include` — and checks whole batch before writing any of it. Covered by `TestUploadNeverReplacesAStatement` and `TestUploadRefusesBadNames`. +**The statements list is read from the folders, not the index.** `/api/files` +walks `importer.StatementFiles` — the same list import reads — and only then +looks each file up in `store.SourceFiles`, comparing `importer.Checksum`, so a +file is listed exactly when import would read it and a file the index remembers +but the disk lost shows as `missing` instead of vanishing. +`/api/files/{account}/{name}` serves a file only by finding it in that list, +never by joining the name onto a path. A statement is served from the app's origin, where a script could drive +the API, so nothing is ever rendered as a page: text is `text/plain` under +`CSP: sandbox`, anything not text or PDF is a sandboxed download, and PDFs — +whose viewers refuse a sandbox — open in the browser's own isolated viewer. +Covered by `TestServeFileServesOnlyStatements`. + ## Adding a bank parser Implement `parser.Parser` and call `parser.Register` from an `init`. Nothing diff --git a/README.md b/README.md index b452b82..c1aca71 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,26 @@ fresh command would: Shift-click **Import** for `import --force`. +### Statements + +The Accounts screen (`1`) also lists every statement file in every account +folder, with what the index made of it: + +| Status | Meaning | +| --- | --- | +| `✓` imported | read, and unchanged since | +| `⚠` changed | the file differs from what was imported; Import re-reads it | +| `•` not imported | new since the last import — or an import tried and failed, in which case the error is in the import report | +| `✗` gone from disk | imported once, then removed; its transactions stay in the index only until it is rebuilt | + +**Added** is how many transactions the file brought in. Statements that overlap +share rows, and a shared row counts towards whichever file was imported first, +so a later statement covering the same days can add fewer rows than it holds. + +Click a file name to open it. PDFs and CSVs open in the browser; anything else +downloads. Only files import would read are listed or served — not +`account.toml`, dotfiles, or anything outside an account's `include` patterns. + ### Adding statements from the browser The Accounts screen (`1`) has an **Add statements** panel: pick the account, diff --git a/internal/importer/importer.go b/internal/importer/importer.go index b9cf7d3..30330b6 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -101,7 +101,7 @@ func Run(root string, db *store.DB, accounts []*config.Account, engine *rules.En continue } - files, err := statementFiles(acc) + files, err := StatementFiles(acc) if err != nil { return res, err } @@ -134,7 +134,7 @@ func importFile(root string, db *store.DB, acc *config.Account, accountID int64, } fr := FileResult{Account: acc.Slug, Path: rel} - sum, err := checksum(path) + sum, err := Checksum(path) if err != nil { fr.Err = err return fr @@ -252,10 +252,11 @@ func fingerprint(key string, ordinal int) string { return hex.EncodeToString(sum[:]) } -// statementFiles lists the files in an account folder that should be parsed: +// StatementFiles lists the files in an account folder that should be parsed: // every regular file except account.toml and dotfiles, narrowed by the -// account's optional include globs. -func statementFiles(acc *config.Account) ([]string, error) { +// account's optional include globs. It is also what the web app lists and +// serves, so a file is shown exactly when import would read it. +func StatementFiles(acc *config.Account) ([]string, error) { entries, err := os.ReadDir(acc.Dir) if err != nil { return nil, fmt.Errorf("read account dir %s: %w", acc.Dir, err) @@ -284,7 +285,9 @@ func matchAny(patterns []string, name string) bool { return false } -func checksum(path string) (string, error) { +// Checksum is the sha256 a statement is recorded under; an import skips a file +// whose checksum has not changed. +func Checksum(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err diff --git a/internal/store/store.go b/internal/store/store.go index 55378cd..b7bb96d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -393,6 +393,42 @@ func (d *DB) Months() ([]string, error) { return out, rows.Err() } +// SourceFileInfo is what the index recorded about one imported statement. +type SourceFileInfo struct { + AccountSlug string + Path string // relative to the data root, as the importer records it + SHA256 string + ImportedAt string + // Added counts the transactions this file introduced. A row seen again in + // an overlapping statement is deduplicated and stays with the file that + // brought it first, so this is not how many rows the file holds. + Added int +} + +// SourceFiles lists every statement the index has imported. +func (d *DB) SourceFiles() ([]SourceFileInfo, error) { + rows, err := d.sql.Query(` + SELECT a.slug, s.path, s.sha256, s.imported_at, COUNT(t.id) + FROM source_files s + JOIN accounts a ON a.id = s.account_id + LEFT JOIN transactions t ON t.source_file_id = s.id + GROUP BY s.id + ORDER BY a.slug, s.path`) + if err != nil { + return nil, fmt.Errorf("list source files: %w", err) + } + defer rows.Close() + var out []SourceFileInfo + for rows.Next() { + var f SourceFileInfo + if err := rows.Scan(&f.AccountSlug, &f.Path, &f.SHA256, &f.ImportedAt, &f.Added); err != nil { + return nil, err + } + out = append(out, f) + } + return out, rows.Err() +} + // Tags lists every tag in use, for completion in the rule builder. func (d *DB) Tags() ([]string, error) { rows, err := d.sql.Query(` diff --git a/internal/web/server.go b/internal/web/server.go index 658bad3..aef35ac 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "io/fs" + "mime" "net/http" "os" "path/filepath" @@ -88,6 +89,8 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("POST /api/import", s.write(s.runImport)) mux.HandleFunc("POST /api/upload", s.write(s.upload)) + mux.HandleFunc("GET /api/files", s.read(s.fileList)) + mux.HandleFunc("GET /api/files/{account}/{name}", s.serveFile) mux.HandleFunc("POST /api/retag", s.write(s.retag)) return mux } @@ -1388,6 +1391,147 @@ func writeStatement(dir string, f uploadFile) error { return nil } +type fileRow struct { + Account string `json:"account"` + Name string `json:"name"` + Size int64 `json:"size"` + // Modified is the file's mtime, YYYY-MM-DD HH:MM in local time; blank for + // a file that is gone from disk. + Modified string `json:"modified"` + // Status is "imported", "changed" (on disk differs from what was + // imported), "new" (not imported yet, or its import failed) or "missing" + // (imported, then removed from disk; its rows stay in the index until it + // is rebuilt). + Status string `json:"status"` + ImportedAt string `json:"importedAt"` + Added int `json:"added"` +} + +// fileList is every statement on disk next to what the index recorded about +// it. The folders are the source of truth, so they decide what is listed, and +// a file the index remembers but the disk no longer holds is called out: its +// rows would not survive a rebuild. +func (s *Server) fileList(*http.Request) (any, error) { + recorded, err := s.db.SourceFiles() + if err != nil { + return nil, err + } + byPath := map[string]store.SourceFileInfo{} + for _, f := range recorded { + byPath[f.Path] = f + } + accounts, err := config.LoadAccounts(s.root) + if err != nil { + return nil, err + } + + out := []fileRow{} + onDisk := map[string]bool{} + for _, acc := range accounts { + paths, err := importer.StatementFiles(acc) + if err != nil { + return nil, err + } + for _, path := range paths { + rel, err := filepath.Rel(s.root, path) + if err != nil { + return nil, err + } + onDisk[rel] = true + row := fileRow{Account: acc.Slug, Name: filepath.Base(path), Status: "new"} + if info, err := os.Stat(path); err == nil { + row.Size, row.Modified = info.Size(), info.ModTime().Format("2006-01-02 15:04") + } + if rec, ok := byPath[rel]; ok { + row.ImportedAt, row.Added, row.Status = rec.ImportedAt, rec.Added, "imported" + if sum, err := importer.Checksum(path); err != nil { + return nil, err + } else if sum != rec.SHA256 { + row.Status = "changed" + } + } + out = append(out, row) + } + } + for _, rec := range recorded { + if !onDisk[rec.Path] { + out = append(out, fileRow{ + Account: rec.AccountSlug, Name: filepath.Base(rec.Path), Status: "missing", + ImportedAt: rec.ImportedAt, Added: rec.Added, + }) + } + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].Account != out[j].Account { + return out[i].Account < out[j].Account + } + return out[i].Name < out[j].Name + }) + return map[string]any{"files": out}, nil +} + +// serveFile opens a statement in the browser. Only a file import would read +// is served — it is looked up in the account's statement list, never joined +// onto a path — so nothing else under the data root can be fetched. +// +// A statement is whatever the bank sent, and it is served from this origin, +// where a script could drive the API; so nothing is ever rendered as a page. +// Text opens inline as text/plain under a sandboxing CSP, and anything that is +// neither text nor PDF downloads. A PDF opens inline without the sandbox: the +// browsers' viewers refuse to run under one, and they render it in their own +// isolated viewer rather than in this origin's DOM. +func (s *Server) serveFile(w http.ResponseWriter, r *http.Request) { + s.mu.RLock() + defer s.mu.RUnlock() + fail := func(code int, msg string) { + writeJSON(w, code, map[string]string{"error": msg}) + } + accounts, err := config.LoadAccounts(s.root) + if err != nil { + fail(http.StatusInternalServerError, err.Error()) + return + } + account, name := r.PathValue("account"), r.PathValue("name") + var path string + for _, acc := range accounts { + if acc.Slug != account { + continue + } + paths, err := importer.StatementFiles(acc) + if err != nil { + fail(http.StatusInternalServerError, err.Error()) + return + } + for _, p := range paths { + if filepath.Base(p) == name { + path = p + } + } + } + if path == "" { + fail(http.StatusNotFound, fmt.Sprintf("no statement %s/%s", account, name)) + return + } + + disposition := "attachment" + switch strings.ToLower(filepath.Ext(name)) { + case ".pdf": + w.Header().Set("Content-Type", "application/pdf") + disposition = "inline" + case ".csv", ".txt", ".tsv": + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.Header().Set("Content-Security-Policy", "sandbox") + disposition = "inline" + default: + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Security-Policy", "sandbox") + } + w.Header().Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": name})) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Cache-Control", "no-store") + http.ServeFile(w, r, path) +} + // retag is `money retag`: both halves of what rules.toml decides, re-derived // from the file as it is on disk now. func (s *Server) retag(*http.Request) (any, error) { diff --git a/internal/web/static/app.js b/internal/web/static/app.js index c2bb57a..442dad0 100644 --- a/internal/web/static/app.js +++ b/internal/web/static/app.js @@ -266,13 +266,15 @@ async function retag() { function mountAccounts(main) { const body = h('div'); + const files = h('div'); const upload = h('div'); - main.append(h('h2', {}, 'Accounts'), body, upload); + main.append(h('h2', {}, 'Accounts'), body, files, upload); async function refresh() { if (!state.overview) await loadOverview(); const o = state.overview; upload.replaceChildren(uploader(o)); + api('GET', '/api/files').then((res) => files.replaceChildren(fileList(res.files))).catch(setError); if (!o.accounts.length) { // An account.toml is not enough on its own: folders reach the index // only through an import, so say that rather than show nothing. @@ -295,6 +297,63 @@ function mountAccounts(main) { return { refresh }; } +const FILE_STATUS = { + imported: ['✓', '', 'imported'], + changed: ['⚠', 'warn', 'changed since it was imported — Import re-reads it'], + new: ['•', 'warn', 'not imported — press Import (a file that fails to parse stays here)'], + missing: ['✗', 'neg', 'gone from disk — its rows stay only until the index is rebuilt'], +}; + +// fileList shows the statements in every account folder beside what the index +// recorded about each, and opens one on click. The folders decide what is +// listed; the index only says how far each file has got. +function fileList(list) { + const panel = h('div', { class: 'panel files' }); + if (!list.length) { + panel.append(h('div', { class: 'empty' }, 'No statement files yet.')); + return h('div', {}, h('h3', { class: 'section' }, 'Statements'), panel); + } + const pending = list.filter((f) => f.status !== 'imported').length; + panel.append(h('table', {}, + h('thead', {}, h('tr', {}, h('th', {}, 'Account'), h('th', {}, 'File'), h('th', { class: 'num' }, 'Size'), + h('th', {}, 'Modified'), h('th', {}, ''), h('th', {}, 'Status'), + h('th', { class: 'num', title: 'Transactions this file brought into the index; rows already brought by an overlapping statement count there' }, 'Added'), + h('th', {}, 'Imported'))), + h('tbody', {}, list.map((f) => { + const [mark, cls, text] = FILE_STATUS[f.status] || ['', '', f.status]; + return h('tr', {}, + h('td', {}, f.account), + h('td', { class: 'desc' }, f.status === 'missing' ? h('span', { class: 'muted' }, f.name) : h('a', { + href: `/api/files/${encodeURIComponent(f.account)}/${encodeURIComponent(f.name)}`, + target: '_blank', rel: 'noopener', title: 'Open this statement', + }, f.name)), + h('td', { class: 'num' }, f.status === 'missing' ? '' : formatSize(f.size)), + h('td', { class: 'muted' }, f.modified), + h('td', { class: 'mark ' + cls }, mark), + h('td', { class: cls || 'muted' }, text), + h('td', { class: 'num' }, f.status === 'new' ? '' : f.added), + h('td', { class: 'muted' }, localTime(f.importedAt))); + })))); + return h('div', {}, h('h3', { class: 'section' }, 'Statements ', + h('span', { class: 'sub' }, `· ${list.length} file(s)` + (pending ? ` · ${pending} need attention` : ''))), panel); +} + +// localTime shows the index's UTC import time in the same local +// YYYY-MM-DD HH:MM that the server gives the files' modified times. +function localTime(iso) { + if (!iso) return ''; + const d = new Date(iso); + if (isNaN(d)) return iso; + const p = (n) => String(n).padStart(2, '0'); + return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}`; +} + +function formatSize(bytes) { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`; + return `${(bytes / 1024 / 1024).toFixed(1)} MB`; +} + // uploader puts statement files into an account folder and imports them. The // folder is where they belong: the index is rebuilt from it, so a statement // that lived only in the index would be lost the next time it is deleted. diff --git a/internal/web/static/style.css b/internal/web/static/style.css index 7a2eb15..d560b6b 100644 --- a/internal/web/static/style.css +++ b/internal/web/static/style.css @@ -179,6 +179,11 @@ td.actions button { padding: 1px 8px; font-size: 12px; } .kept { font-size: 12px; color: var(--muted); font-family: var(--mono); } .preview { max-height: calc(100vh - 170px); } +h3.section { font-size: 14px; font-weight: 600; margin: 20px 0 8px; } +h3.section .sub { color: var(--muted); font-weight: 400; } +.files a { color: var(--accent); text-decoration: none; } +.files a:hover { text-decoration: underline; } + .upload { margin-top: 16px; padding: 14px; overflow: visible; } .upload h3 { font-size: 14px; margin: 0 0 10px; } .upload .toolbar { margin-bottom: 10px; } diff --git a/internal/web/upload_test.go b/internal/web/upload_test.go index 1fc9100..d25a905 100644 --- a/internal/web/upload_test.go +++ b/internal/web/upload_test.go @@ -164,3 +164,90 @@ func TestUploadRefusesMultipart(t *testing.T) { t.Errorf("status %d, want %d", w.Code, http.StatusUnsupportedMediaType) } } + +// The statements list is the folders, each beside what the index recorded: +// imported, changed since, not imported yet, and gone from disk. +func TestFileListStatuses(t *testing.T) { + dir, h := newUploadServer(t, checkingTOML) + write := func(name, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("a.csv", statement) + write("b.csv", "date,description,amount\n2026-03-01,ZARA,-40.00\n") + write("c.csv", "date,description,amount\n2026-04-01,KAUFLAND,-9.00\n") + call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil) + + write("b.csv", "date,description,amount\n2026-03-01,ZARA,-41.00\n") + if err := os.Remove(filepath.Join(dir, "c.csv")); err != nil { + t.Fatal(err) + } + write("d.csv", statement) + write(".hidden.csv", statement) + + var res struct{ Files []fileRow } + call(t, h, "GET", "/api/files", nil, http.StatusOK, &res) + got := map[string]fileRow{} + var names []string + for _, f := range res.Files { + got[f.Name] = f + names = append(names, f.Name) + } + if strings.Join(names, " ") != "a.csv b.csv c.csv d.csv" { + t.Fatalf("files = %v, want the four statements and nothing import would skip", names) + } + for name, want := range map[string]string{"a.csv": "imported", "b.csv": "changed", "c.csv": "missing", "d.csv": "new"} { + if got[name].Status != want { + t.Errorf("%s status = %q, want %q", name, got[name].Status, want) + } + } + if got["a.csv"].Added != 2 || got["c.csv"].Added != 1 || got["a.csv"].ImportedAt == "" { + t.Errorf("a.csv = %+v, c.csv = %+v", got["a.csv"], got["c.csv"]) + } + if got["a.csv"].Size != int64(len(statement)) { + t.Errorf("a.csv size = %d, want %d", got["a.csv"].Size, len(statement)) + } +} + +// Only a file import would read is served, and never as a page. +func TestServeFileServesOnlyStatements(t *testing.T) { + dir, h := newUploadServer(t, checkingTOML) + for name, body := range map[string]string{"a.csv": statement, "page.html": "", ".secret": "x"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + get := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest("GET", path, nil)) + return w + } + + w := get("/api/files/checking/a.csv") + if w.Code != http.StatusOK || w.Body.String() != statement { + t.Fatalf("a.csv: %d %q", w.Code, w.Body.String()) + } + if ct := w.Header().Get("Content-Type"); ct != "text/plain; charset=utf-8" { + t.Errorf("a.csv content type = %q", ct) + } + + w = get("/api/files/checking/page.html") + if w.Code != http.StatusOK || w.Header().Get("Content-Type") != "application/octet-stream" || + w.Header().Get("Content-Security-Policy") != "sandbox" || + !strings.HasPrefix(w.Header().Get("Content-Disposition"), "attachment") { + t.Errorf("page.html served as %d %v, want a sandboxed download", w.Code, w.Header()) + } + + for _, path := range []string{ + "/api/files/checking/" + config.AccountFile, + "/api/files/checking/.secret", + "/api/files/checking/..%2F" + config.RulesFile, + "/api/files/nope/a.csv", + } { + if w := get(path); w.Code != http.StatusNotFound { + t.Errorf("%s: status %d, want 404", path, w.Code) + } + } +}