nikolaandClaude Opus 5.5
a5f7541980
Four changes to statement handling in the web app, made together and touching the same upload and statements-list code. Name NLB uploads by statement date. parser.Namer is an optional interface, like Warner, through which a parser names its statements; nlb reads the "Datum izpiska" from the izpisek header and names it izpisek_YYYY_MM_DD, lowercase, extension included -- ported from the rename_izpiski.py it replaces. Uploads are staged as dotfiles, invisible to import, so the parser can read them; two downloads of one statement then meet under one name and the second is recognised as already there, while a different statement of the same date is numbered _2 as the script did. Only uploads are named: source_files records statements by path, so renaming a file already in a folder would orphan its rows. Delete a statement from the statements list. The file is removed from disk for good -- the page says so before it asks -- and store.ForgetSourceFile drops its transactions and their transfer rows. A row two overlapping statements share is stored once, under the file imported first, so it goes too; the account's other statements forget their checksums and show as changed until the next Import re-reads them and restores it. A file already gone from disk can be forgotten. Upload and delete no longer import. Importing stays the user's call, made with the Import button, so a batch can be put together and looked over first. Delete still re-pairs transfers, which reads no statement. Show rows and new rows per statement. The list read "0" for a file whose rows an earlier, overlapping statement already held, which looked like a file that failed to parse. source_files now records how many transactions each statement holds, and the list reads "3 rows · 0 new". This adds a column the code reads, so an index built by an earlier version fails with "no such column: s.rows": delete index.db and import again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
443 lines
17 KiB
Go
443 lines
17 KiB
Go
package web
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.petrovv.com/nikola/money/internal/config"
|
|
"git.petrovv.com/nikola/money/internal/parser"
|
|
"git.petrovv.com/nikola/money/internal/rules"
|
|
"git.petrovv.com/nikola/money/internal/store"
|
|
"git.petrovv.com/nikola/money/internal/transfers"
|
|
)
|
|
|
|
// The upload tests are about where files land and what is refused, not about
|
|
// any bank's layout, so they read "date,description,amount" with a header.
|
|
func init() {
|
|
parser.Register("webtest", func(acc *config.Account) (parser.Parser, error) {
|
|
return csvParser{digits: acc.Digits()}, nil
|
|
})
|
|
parser.Register("webnamed", func(acc *config.Account) (parser.Parser, error) {
|
|
return namingParser{csvParser{digits: acc.Digits()}}, nil
|
|
})
|
|
}
|
|
|
|
type csvParser struct{ digits int }
|
|
|
|
// namingParser is csvParser for a bank whose downloads are named unhelpfully:
|
|
// it names a statement after its first date, as nlb does after the statement
|
|
// date, and says nothing for a statement with no rows.
|
|
type namingParser struct{ csvParser }
|
|
|
|
func (namingParser) StatementName(path string) (string, error) {
|
|
body, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
lines := strings.Split(strings.TrimSpace(string(body)), "\n")
|
|
if len(lines) < 2 {
|
|
return "", nil
|
|
}
|
|
return "stmt_" + strings.Split(lines[1], ",")[0], nil
|
|
}
|
|
|
|
func (p csvParser) Parse(path string, _ *config.Account) ([]parser.RawTxn, error) {
|
|
body, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var txns []parser.RawTxn
|
|
for i, line := range strings.Split(strings.TrimSpace(string(body)), "\n")[1:] {
|
|
f := strings.Split(line, ",")
|
|
if len(f) != 3 {
|
|
return nil, fmt.Errorf("row %d: want 3 fields", i+2)
|
|
}
|
|
amount, err := parser.ParseAmount(f[2], ".", "", p.digits)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
txns = append(txns, parser.RawTxn{Date: f[0], Description: f[1], AmountMinor: amount})
|
|
}
|
|
return txns, nil
|
|
}
|
|
|
|
// newUploadServer builds a server over a data root with one empty account
|
|
// folder, as `money serve` sees it before the first import.
|
|
func newUploadServer(t *testing.T, accountTOML string) (string, http.Handler) {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
dir := filepath.Join(root, "checking")
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, config.AccountFile), []byte(accountTOML), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db, err := store.Open(config.IndexPath(root))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
accounts, err := config.LoadAccounts(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
loaded, err := config.LoadRules(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s := New(root, db, accounts, rules.New(loaded), transfers.New(loaded))
|
|
s.now = func() time.Time { return time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC) }
|
|
return dir, s.Handler()
|
|
}
|
|
|
|
const checkingTOML = "currency = \"EUR\"\nparser = \"webtest\"\n"
|
|
|
|
const statement = "date,description,amount\n2026-02-01,LIDL SOFIA,-12.50\n2026-02-03,SALARY,1000.00\n"
|
|
|
|
// An upload only saves: the file waits on the statements list as new until
|
|
// the user presses Import.
|
|
func TestUploadSavesWithoutImporting(t *testing.T) {
|
|
dir, h := newUploadServer(t, checkingTOML)
|
|
req := uploadReq{Account: "checking", Files: []uploadFile{{Name: "2026-02.csv", Data: []byte(statement)}}}
|
|
|
|
var res status
|
|
call(t, h, "POST", "/api/upload", req, http.StatusOK, &res)
|
|
if res.Status != "uploaded 1 file(s) to checking · press Import to read them" {
|
|
t.Errorf("status = %q", res.Status)
|
|
}
|
|
if got, err := os.ReadFile(filepath.Join(dir, "2026-02.csv")); err != nil || string(got) != statement {
|
|
t.Errorf("file on disk = %q, %v", got, err)
|
|
}
|
|
if got := descriptions(t, h); got != "" {
|
|
t.Errorf("upload imported %s; it should only save", got)
|
|
}
|
|
var files struct{ Files []fileRow }
|
|
call(t, h, "GET", "/api/files", nil, http.StatusOK, &files)
|
|
if len(files.Files) != 1 || files.Files[0].Status != "new" {
|
|
t.Errorf("files = %+v, want the upload waiting as new", files.Files)
|
|
}
|
|
|
|
call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
|
|
if got := descriptions(t, h); got != "SALARY,LIDL SOFIA" {
|
|
t.Errorf("after Import the index holds %s", got)
|
|
}
|
|
|
|
// The same file again is not an error, and saves nothing.
|
|
call(t, h, "POST", "/api/upload", req, http.StatusOK, &res)
|
|
if res.Status != "uploaded 0 file(s) to checking (1 already there)" {
|
|
t.Errorf("re-upload status = %q", res.Status)
|
|
}
|
|
}
|
|
|
|
// A statement already in the folder is the source of truth for what it
|
|
// imported, so an upload never replaces one with different contents.
|
|
func TestUploadNeverReplacesAStatement(t *testing.T) {
|
|
dir, h := newUploadServer(t, checkingTOML)
|
|
path := filepath.Join(dir, "2026-02.csv")
|
|
if err := os.WriteFile(path, []byte(statement), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
req := uploadReq{Account: "checking", Files: []uploadFile{{Name: "2026-02.csv", Data: []byte("date,description,amount\n")}}}
|
|
call(t, h, "POST", "/api/upload", req, http.StatusConflict, nil)
|
|
if got, _ := os.ReadFile(path); string(got) != statement {
|
|
t.Errorf("statement was overwritten: %q", got)
|
|
}
|
|
}
|
|
|
|
// A batch is checked as a whole before anything is written, and nothing may
|
|
// land outside the account folder or where import would not read it back.
|
|
func TestUploadRefusesBadNames(t *testing.T) {
|
|
dir, h := newUploadServer(t, checkingTOML+"include = [\"*.csv\"]\n")
|
|
for _, bad := range []string{"../escape.csv", "sub/x.csv", ".hidden.csv", config.AccountFile, "notes.txt", ""} {
|
|
req := uploadReq{Account: "checking", Files: []uploadFile{
|
|
{Name: "good.csv", Data: []byte(statement)},
|
|
{Name: bad, Data: []byte(statement)},
|
|
}}
|
|
call(t, h, "POST", "/api/upload", req, http.StatusBadRequest, nil)
|
|
}
|
|
entries, _ := os.ReadDir(dir)
|
|
if len(entries) != 1 {
|
|
var names []string
|
|
for _, e := range entries {
|
|
names = append(names, e.Name())
|
|
}
|
|
t.Errorf("folder holds %v, want only %s", names, config.AccountFile)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(filepath.Dir(dir), "escape.csv")); err == nil {
|
|
t.Error("a file escaped the account folder")
|
|
}
|
|
|
|
call(t, h, "POST", "/api/upload", uploadReq{Account: "nope", Files: []uploadFile{{Name: "a.csv"}}},
|
|
http.StatusBadRequest, nil)
|
|
}
|
|
|
|
// Multipart is what a cross-site form can send, so it is refused like any
|
|
// other non-JSON write.
|
|
func TestUploadRefusesMultipart(t *testing.T) {
|
|
_, h := newUploadServer(t, checkingTOML)
|
|
r := httptest.NewRequest("POST", "/api/upload", strings.NewReader("--x\r\n"))
|
|
r.Header.Set("Content-Type", "multipart/form-data; boundary=x")
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, r)
|
|
if w.Code != http.StatusUnsupportedMediaType {
|
|
t.Errorf("status %d, want %d", w.Code, http.StatusUnsupportedMediaType)
|
|
}
|
|
}
|
|
|
|
// The statements list is the folders, each beside what the index recorded:
|
|
// imported, changed since, not imported yet, and gone from disk.
|
|
func TestFileListStatuses(t *testing.T) {
|
|
dir, h := newUploadServer(t, checkingTOML)
|
|
write := func(name, body string) {
|
|
t.Helper()
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
write("a.csv", statement)
|
|
write("b.csv", "date,description,amount\n2026-03-01,ZARA,-40.00\n")
|
|
write("c.csv", "date,description,amount\n2026-04-01,KAUFLAND,-9.00\n")
|
|
call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
|
|
|
|
write("b.csv", "date,description,amount\n2026-03-01,ZARA,-41.00\n")
|
|
if err := os.Remove(filepath.Join(dir, "c.csv")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
write("d.csv", statement)
|
|
write(".hidden.csv", statement)
|
|
|
|
var res struct{ Files []fileRow }
|
|
call(t, h, "GET", "/api/files", nil, http.StatusOK, &res)
|
|
got := map[string]fileRow{}
|
|
var names []string
|
|
for _, f := range res.Files {
|
|
got[f.Name] = f
|
|
names = append(names, f.Name)
|
|
}
|
|
if strings.Join(names, " ") != "a.csv b.csv c.csv d.csv" {
|
|
t.Fatalf("files = %v, want the four statements and nothing import would skip", names)
|
|
}
|
|
for name, want := range map[string]string{"a.csv": "imported", "b.csv": "changed", "c.csv": "missing", "d.csv": "new"} {
|
|
if got[name].Status != want {
|
|
t.Errorf("%s status = %q, want %q", name, got[name].Status, want)
|
|
}
|
|
}
|
|
if got["a.csv"].Rows != 2 || got["a.csv"].Added != 2 || got["c.csv"].Rows != 1 || got["c.csv"].Added != 1 ||
|
|
got["a.csv"].ImportedAt == "" {
|
|
t.Errorf("a.csv = %+v, c.csv = %+v", got["a.csv"], got["c.csv"])
|
|
}
|
|
if got["a.csv"].Size != int64(len(statement)) {
|
|
t.Errorf("a.csv size = %d, want %d", got["a.csv"].Size, len(statement))
|
|
}
|
|
}
|
|
|
|
// Only a file import would read is served, and never as a page.
|
|
func TestServeFileServesOnlyStatements(t *testing.T) {
|
|
dir, h := newUploadServer(t, checkingTOML)
|
|
for name, body := range map[string]string{"a.csv": statement, "page.html": "<script>alert(1)</script>", ".secret": "x"} {
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
get := func(path string) *httptest.ResponseRecorder {
|
|
w := httptest.NewRecorder()
|
|
h.ServeHTTP(w, httptest.NewRequest("GET", path, nil))
|
|
return w
|
|
}
|
|
|
|
w := get("/api/files/checking/a.csv")
|
|
if w.Code != http.StatusOK || w.Body.String() != statement {
|
|
t.Fatalf("a.csv: %d %q", w.Code, w.Body.String())
|
|
}
|
|
if ct := w.Header().Get("Content-Type"); ct != "text/plain; charset=utf-8" {
|
|
t.Errorf("a.csv content type = %q", ct)
|
|
}
|
|
|
|
w = get("/api/files/checking/page.html")
|
|
if w.Code != http.StatusOK || w.Header().Get("Content-Type") != "application/octet-stream" ||
|
|
w.Header().Get("Content-Security-Policy") != "sandbox" ||
|
|
!strings.HasPrefix(w.Header().Get("Content-Disposition"), "attachment") {
|
|
t.Errorf("page.html served as %d %v, want a sandboxed download", w.Code, w.Header())
|
|
}
|
|
|
|
for _, path := range []string{
|
|
"/api/files/checking/" + config.AccountFile,
|
|
"/api/files/checking/.secret",
|
|
"/api/files/checking/..%2F" + config.RulesFile,
|
|
"/api/files/nope/a.csv",
|
|
} {
|
|
if w := get(path); w.Code != http.StatusNotFound {
|
|
t.Errorf("%s: status %d, want 404", path, w.Code)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A parser that names its statements decides the stored name, so the bank's
|
|
// "download (3).pdf" lands as something that sorts by date. A reissue of the
|
|
// same date is numbered rather than refused, and a statement that does not
|
|
// say keeps the name it came with.
|
|
func TestUploadNamesStatementsTheParserCanName(t *testing.T) {
|
|
dir, h := newUploadServer(t, "currency = \"EUR\"\nparser = \"webnamed\"\n")
|
|
upload := func(files ...uploadFile) string {
|
|
t.Helper()
|
|
var res status
|
|
call(t, h, "POST", "/api/upload", uploadReq{Account: "checking", Files: files}, http.StatusOK, &res)
|
|
return res.Status
|
|
}
|
|
reissue := "date,description,amount\n2026-02-01,LIDL SOFIA,-12.50\n2026-02-04,ZARA,-30.00\n"
|
|
other := "date,description,amount\n2026-03-01,KAUFLAND,-9.00\n"
|
|
|
|
status := upload(uploadFile{Name: "download (3).csv", Data: []byte(statement)})
|
|
if !strings.Contains(status, "download (3).csv → stmt_2026-02-01.csv") {
|
|
t.Errorf("status = %q, want the rename named", status)
|
|
}
|
|
// The same statement downloaded again is recognised under its new name.
|
|
if status := upload(uploadFile{Name: "download (4).csv", Data: []byte(statement)}); !strings.Contains(status, "1 already there") {
|
|
t.Errorf("second download: status = %q", status)
|
|
}
|
|
// A different statement of the same date, and two of one date in a batch.
|
|
upload(uploadFile{Name: "x.csv", Data: []byte(reissue)},
|
|
uploadFile{Name: "y.csv", Data: []byte(reissue + "2026-02-05,BOLT,-4.00\n")},
|
|
uploadFile{Name: "Z.CSV", Data: []byte(other)}, // a chosen name is lowercase
|
|
uploadFile{Name: "empty.csv", Data: []byte("date,description,amount\n")})
|
|
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var names []string
|
|
for _, e := range entries {
|
|
names = append(names, e.Name())
|
|
}
|
|
want := config.AccountFile + " empty.csv stmt_2026-02-01.csv stmt_2026-02-01_2.csv stmt_2026-02-01_3.csv stmt_2026-03-01.csv"
|
|
if strings.Join(names, " ") != want {
|
|
t.Errorf("folder = %v, want %s", names, want)
|
|
}
|
|
}
|
|
|
|
func writeFile(t *testing.T, path, body string) {
|
|
t.Helper()
|
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func descriptions(t *testing.T, h http.Handler) string {
|
|
t.Helper()
|
|
var txns struct{ Rows []txnRow }
|
|
call(t, h, "GET", "/api/transactions", nil, http.StatusOK, &txns)
|
|
var out []string
|
|
for _, r := range txns.Rows {
|
|
out = append(out, r.Description)
|
|
}
|
|
return strings.Join(out, ",")
|
|
}
|
|
|
|
// Removing a statement deletes the file and takes out what it brought in. A
|
|
// row an overlapping statement also holds was stored under the file imported
|
|
// first; the next Import brings it back from the other one.
|
|
func TestRemoveFileKeepsWhatAnotherStatementHolds(t *testing.T) {
|
|
dir, h := newUploadServer(t, checkingTOML)
|
|
writeFile(t, filepath.Join(dir, "a.csv"), "date,description,amount\n2026-01-30,ONLY IN A,-1.00\n2026-02-01,SHARED,-2.00\n")
|
|
writeFile(t, filepath.Join(dir, "b.csv"), "date,description,amount\n2026-02-01,SHARED,-2.00\n2026-02-03,ONLY IN B,-3.00\n")
|
|
call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
|
|
|
|
// Each holds two rows; b.csv was imported second, so its shared row
|
|
// stayed with a.csv and it brought in only one.
|
|
var before struct{ Files []fileRow }
|
|
call(t, h, "GET", "/api/files", nil, http.StatusOK, &before)
|
|
if f := before.Files; len(f) != 2 || f[0].Rows != 2 || f[0].Added != 2 || f[1].Rows != 2 || f[1].Added != 1 {
|
|
t.Errorf("files = %+v, want a.csv 2 rows · 2 new and b.csv 2 rows · 1 new", f)
|
|
}
|
|
|
|
var res status
|
|
call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "a.csv"}, http.StatusOK, &res)
|
|
if !strings.HasPrefix(res.Status, "deleted checking/a.csv, 2 transaction(s) dropped") {
|
|
t.Errorf("status = %q", res.Status)
|
|
}
|
|
// Deleting does not import, so the shared row is gone for now, and b.csv
|
|
// says it is due to be read again.
|
|
if got := descriptions(t, h); got != "ONLY IN B" {
|
|
t.Errorf("index holds %s right after the delete, want only b's own row", got)
|
|
}
|
|
var files struct{ Files []fileRow }
|
|
call(t, h, "GET", "/api/files", nil, http.StatusOK, &files)
|
|
if len(files.Files) != 1 || files.Files[0].Status != "changed" {
|
|
t.Errorf("files = %+v, want b.csv marked to be re-read", files.Files)
|
|
}
|
|
|
|
call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
|
|
if got := descriptions(t, h); got != "ONLY IN B,SHARED" {
|
|
t.Errorf("after Import the index holds %s, want the shared row restored from b", got)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "a.csv")); !errors.Is(err, fs.ErrNotExist) {
|
|
t.Errorf("a.csv is still on disk: %v", err)
|
|
}
|
|
if entries, _ := os.ReadDir(dir); len(entries) != 2 {
|
|
t.Errorf("folder holds %d entries, want account.toml and b.csv only", len(entries))
|
|
}
|
|
|
|
call(t, h, "GET", "/api/files", nil, http.StatusOK, &files)
|
|
if len(files.Files) != 1 || files.Files[0].Name != "b.csv" || files.Files[0].Status != "imported" {
|
|
t.Errorf("files = %+v, want only b.csv, imported", files.Files)
|
|
}
|
|
}
|
|
|
|
// A removed leg takes its transfer pairing with it, and the other leg is left
|
|
// unpaired — which is the truth once one side is gone.
|
|
func TestRemoveFileUnpairsItsTransfers(t *testing.T) {
|
|
dir, h := newUploadServer(t, checkingTOML)
|
|
writeFile(t, filepath.Join(filepath.Dir(dir), config.RulesFile), `
|
|
[[transfer]]
|
|
from_account = "checking"
|
|
from_desc = "*OUT*"
|
|
to_account = "checking"
|
|
to_desc = "*IN*"
|
|
`)
|
|
writeFile(t, filepath.Join(dir, "out.csv"), "date,description,amount\n2026-02-01,MOVE OUT,-5.00\n")
|
|
writeFile(t, filepath.Join(dir, "in.csv"), "date,description,amount\n2026-02-02,MOVE IN,5.00\n")
|
|
call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
|
|
|
|
var res status
|
|
call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "in.csv"}, http.StatusOK, &res)
|
|
if !strings.Contains(res.Status, "1 transfer leg(s) unpaired") {
|
|
t.Errorf("status = %q, want the remaining leg reported unpaired", res.Status)
|
|
}
|
|
}
|
|
|
|
// A file already gone from disk is forgotten by the index; a name the index
|
|
// never had is not found.
|
|
func TestRemoveFileForgetsAMissingOne(t *testing.T) {
|
|
dir, h := newUploadServer(t, checkingTOML)
|
|
writeFile(t, filepath.Join(dir, "a.csv"), statement)
|
|
call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
|
|
if err := os.Remove(filepath.Join(dir, "a.csv")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var res status
|
|
call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "a.csv"}, http.StatusOK, &res)
|
|
if !strings.HasPrefix(res.Status, "forgot checking/a.csv, 2 transaction(s) dropped") {
|
|
t.Errorf("status = %q", res.Status)
|
|
}
|
|
if got := descriptions(t, h); got != "" {
|
|
t.Errorf("index still holds %s", got)
|
|
}
|
|
|
|
call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "a.csv"}, http.StatusNotFound, nil)
|
|
call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "../rules.toml"}, http.StatusBadRequest, nil)
|
|
if _, err := os.Stat(filepath.Join(dir, config.AccountFile)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: config.AccountFile}, http.StatusNotFound, nil)
|
|
}
|