Name NLB uploads, delete statements, and stop importing on upload

Four changes to statement handling in the web app, made together and touching the same upload and statements-list code. Name NLB uploads by statement date. parser.Namer is an optional interface, like Warner, through which a parser names its statements; nlb reads the "Datum izpiska" from the izpisek header and names it izpisek_YYYY_MM_DD, lowercase, extension included -- ported from the rename_izpiski.py it replaces. Uploads are staged as dotfiles, invisible to import, so the parser can read them; two downloads of one statement then meet under one name and the second is recognised as already there, while a different statement of the same date is numbered _2 as the script did. Only uploads are named: source_files records statements by path, so renaming a file already in a folder would orphan its rows. Delete a statement from the statements list. The file is removed from disk for good -- the page says so before it asks -- and store.ForgetSourceFile drops its transactions and their transfer rows. A row two overlapping statements share is stored once, under the file imported first, so it goes too; the account's other statements forget their checksums and show as changed until the next Import re-reads them and restores it. A file already gone from disk can be forgotten. Upload and delete no longer import. Importing stays the user's call, made with the Import button, so a batch can be put together and looked over first. Delete still re-pairs transfers, which reads no statement. Show rows and new rows per statement. The list read "0" for a file whose rows an earlier, overlapping statement already held, which looked like a file that failed to parse. source_files now records how many transactions each statement holds, and the list reads "3 rows · 0 new". This adds a column the code reads, so an index built by an earlier version fails with "no such column: s.rows": delete index.db and import again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-02 21:48:29 +02:00
parent 9ad6c05f9a
commit a5f7541980
12 changed files with 652 additions and 146 deletions
@@ -1,7 +1,9 @@
 package web

 import (
+	"errors"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -23,10 +25,30 @@ func init() {
 	parser.Register("webtest", func(acc *config.Account) (parser.Parser, error) {
 		return csvParser{digits: acc.Digits()}, nil
 	})
+	parser.Register("webnamed", func(acc *config.Account) (parser.Parser, error) {
+		return namingParser{csvParser{digits: acc.Digits()}}, nil
+	})
 }

 type csvParser struct{ digits int }

+// namingParser is csvParser for a bank whose downloads are named unhelpfully:
+// it names a statement after its first date, as nlb does after the statement
+// date, and says nothing for a statement with no rows.
+type namingParser struct{ csvParser }
+
+func (namingParser) StatementName(path string) (string, error) {
+	body, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	lines := strings.Split(strings.TrimSpace(string(body)), "\n")
+	if len(lines) < 2 {
+		return "", nil
+	}
+	return "stmt_" + strings.Split(lines[1], ",")[0], nil
+}
+
 func (p csvParser) Parse(path string, _ *config.Account) ([]parser.RawTxn, error) {
 	body, err := os.ReadFile(path)
 	if err != nil {
@@ -81,32 +103,38 @@ const checkingTOML = "currency = \"EUR\"\nparser = \"webtest\"\n"

 const statement = "date,description,amount\n2026-02-01,LIDL SOFIA,-12.50\n2026-02-03,SALARY,1000.00\n"

-func TestUploadSavesAndImports(t *testing.T) {
+// An upload only saves: the file waits on the statements list as new until
+// the user presses Import.
+func TestUploadSavesWithoutImporting(t *testing.T) {
 	dir, h := newUploadServer(t, checkingTOML)
 	req := uploadReq{Account: "checking", Files: []uploadFile{{Name: "2026-02.csv", Data: []byte(statement)}}}

-	var res importJSON
+	var res status
 	call(t, h, "POST", "/api/upload", req, http.StatusOK, &res)
-	if !strings.Contains(res.Status, "uploaded 1 file(s) to checking") || !strings.Contains(res.Status, "2 new") {
+	if res.Status != "uploaded 1 file(s) to checking · press Import to read them" {
 		t.Errorf("status = %q", res.Status)
 	}
 	if got, err := os.ReadFile(filepath.Join(dir, "2026-02.csv")); err != nil || string(got) != statement {
 		t.Errorf("file on disk = %q, %v", got, err)
 	}
-	var txns struct{ Rows []txnRow }
-	call(t, h, "GET", "/api/transactions", nil, http.StatusOK, &txns)
-	if len(txns.Rows) != 2 {
-		t.Errorf("index holds %d rows, want the 2 uploaded", len(txns.Rows))
+	if got := descriptions(t, h); got != "" {
+		t.Errorf("upload imported %s; it should only save", got)
+	}
+	var files struct{ Files []fileRow }
+	call(t, h, "GET", "/api/files", nil, http.StatusOK, &files)
+	if len(files.Files) != 1 || files.Files[0].Status != "new" {
+		t.Errorf("files = %+v, want the upload waiting as new", files.Files)
 	}

-	// The same file again is not an error, and adds nothing.
-	call(t, h, "POST", "/api/upload", req, http.StatusOK, &res)
-	if !strings.Contains(res.Status, "uploaded 0 file(s)") || !strings.Contains(res.Status, "1 already there") {
-		t.Errorf("re-upload status = %q", res.Status)
+	call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
+	if got := descriptions(t, h); got != "SALARY,LIDL SOFIA" {
+		t.Errorf("after Import the index holds %s", got)
 	}
-	call(t, h, "GET", "/api/transactions", nil, http.StatusOK, &txns)
-	if len(txns.Rows) != 2 {
-		t.Errorf("re-upload left %d rows, want 2", len(txns.Rows))
+
+	// The same file again is not an error, and saves nothing.
+	call(t, h, "POST", "/api/upload", req, http.StatusOK, &res)
+	if res.Status != "uploaded 0 file(s) to checking (1 already there)" {
+		t.Errorf("re-upload status = %q", res.Status)
 	}
 }

@@ -203,7 +231,8 @@ func TestFileListStatuses(t *testing.T) {
 			t.Errorf("%s status = %q, want %q", name, got[name].Status, want)
 		}
 	}
-	if got["a.csv"].Added != 2 || got["c.csv"].Added != 1 || got["a.csv"].ImportedAt == "" {
+	if got["a.csv"].Rows != 2 || got["a.csv"].Added != 2 || got["c.csv"].Rows != 1 || got["c.csv"].Added != 1 ||
+		got["a.csv"].ImportedAt == "" {
 		t.Errorf("a.csv = %+v, c.csv = %+v", got["a.csv"], got["c.csv"])
 	}
 	if got["a.csv"].Size != int64(len(statement)) {
@@ -251,3 +280,163 @@ func TestServeFileServesOnlyStatements(t *testing.T) {
 		}
 	}
 }
+
+// A parser that names its statements decides the stored name, so the bank's
+// "download (3).pdf" lands as something that sorts by date. A reissue of the
+// same date is numbered rather than refused, and a statement that does not
+// say keeps the name it came with.
+func TestUploadNamesStatementsTheParserCanName(t *testing.T) {
+	dir, h := newUploadServer(t, "currency = \"EUR\"\nparser = \"webnamed\"\n")
+	upload := func(files ...uploadFile) string {
+		t.Helper()
+		var res status
+		call(t, h, "POST", "/api/upload", uploadReq{Account: "checking", Files: files}, http.StatusOK, &res)
+		return res.Status
+	}
+	reissue := "date,description,amount\n2026-02-01,LIDL SOFIA,-12.50\n2026-02-04,ZARA,-30.00\n"
+	other := "date,description,amount\n2026-03-01,KAUFLAND,-9.00\n"
+
+	status := upload(uploadFile{Name: "download (3).csv", Data: []byte(statement)})
+	if !strings.Contains(status, "download (3).csv → stmt_2026-02-01.csv") {
+		t.Errorf("status = %q, want the rename named", status)
+	}
+	// The same statement downloaded again is recognised under its new name.
+	if status := upload(uploadFile{Name: "download (4).csv", Data: []byte(statement)}); !strings.Contains(status, "1 already there") {
+		t.Errorf("second download: status = %q", status)
+	}
+	// A different statement of the same date, and two of one date in a batch.
+	upload(uploadFile{Name: "x.csv", Data: []byte(reissue)},
+		uploadFile{Name: "y.csv", Data: []byte(reissue + "2026-02-05,BOLT,-4.00\n")},
+		uploadFile{Name: "Z.CSV", Data: []byte(other)}, // a chosen name is lowercase
+		uploadFile{Name: "empty.csv", Data: []byte("date,description,amount\n")})
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var names []string
+	for _, e := range entries {
+		names = append(names, e.Name())
+	}
+	want := config.AccountFile + " empty.csv stmt_2026-02-01.csv stmt_2026-02-01_2.csv stmt_2026-02-01_3.csv stmt_2026-03-01.csv"
+	if strings.Join(names, " ") != want {
+		t.Errorf("folder = %v, want %s", names, want)
+	}
+}
+
+func writeFile(t *testing.T, path, body string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func descriptions(t *testing.T, h http.Handler) string {
+	t.Helper()
+	var txns struct{ Rows []txnRow }
+	call(t, h, "GET", "/api/transactions", nil, http.StatusOK, &txns)
+	var out []string
+	for _, r := range txns.Rows {
+		out = append(out, r.Description)
+	}
+	return strings.Join(out, ",")
+}
+
+// Removing a statement deletes the file and takes out what it brought in. A
+// row an overlapping statement also holds was stored under the file imported
+// first; the next Import brings it back from the other one.
+func TestRemoveFileKeepsWhatAnotherStatementHolds(t *testing.T) {
+	dir, h := newUploadServer(t, checkingTOML)
+	writeFile(t, filepath.Join(dir, "a.csv"), "date,description,amount\n2026-01-30,ONLY IN A,-1.00\n2026-02-01,SHARED,-2.00\n")
+	writeFile(t, filepath.Join(dir, "b.csv"), "date,description,amount\n2026-02-01,SHARED,-2.00\n2026-02-03,ONLY IN B,-3.00\n")
+	call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
+
+	// Each holds two rows; b.csv was imported second, so its shared row
+	// stayed with a.csv and it brought in only one.
+	var before struct{ Files []fileRow }
+	call(t, h, "GET", "/api/files", nil, http.StatusOK, &before)
+	if f := before.Files; len(f) != 2 || f[0].Rows != 2 || f[0].Added != 2 || f[1].Rows != 2 || f[1].Added != 1 {
+		t.Errorf("files = %+v, want a.csv 2 rows · 2 new and b.csv 2 rows · 1 new", f)
+	}
+
+	var res status
+	call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "a.csv"}, http.StatusOK, &res)
+	if !strings.HasPrefix(res.Status, "deleted checking/a.csv, 2 transaction(s) dropped") {
+		t.Errorf("status = %q", res.Status)
+	}
+	// Deleting does not import, so the shared row is gone for now, and b.csv
+	// says it is due to be read again.
+	if got := descriptions(t, h); got != "ONLY IN B" {
+		t.Errorf("index holds %s right after the delete, want only b's own row", got)
+	}
+	var files struct{ Files []fileRow }
+	call(t, h, "GET", "/api/files", nil, http.StatusOK, &files)
+	if len(files.Files) != 1 || files.Files[0].Status != "changed" {
+		t.Errorf("files = %+v, want b.csv marked to be re-read", files.Files)
+	}
+
+	call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
+	if got := descriptions(t, h); got != "ONLY IN B,SHARED" {
+		t.Errorf("after Import the index holds %s, want the shared row restored from b", got)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "a.csv")); !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("a.csv is still on disk: %v", err)
+	}
+	if entries, _ := os.ReadDir(dir); len(entries) != 2 {
+		t.Errorf("folder holds %d entries, want account.toml and b.csv only", len(entries))
+	}
+
+	call(t, h, "GET", "/api/files", nil, http.StatusOK, &files)
+	if len(files.Files) != 1 || files.Files[0].Name != "b.csv" || files.Files[0].Status != "imported" {
+		t.Errorf("files = %+v, want only b.csv, imported", files.Files)
+	}
+}
+
+// A removed leg takes its transfer pairing with it, and the other leg is left
+// unpaired — which is the truth once one side is gone.
+func TestRemoveFileUnpairsItsTransfers(t *testing.T) {
+	dir, h := newUploadServer(t, checkingTOML)
+	writeFile(t, filepath.Join(filepath.Dir(dir), config.RulesFile), `
+[[transfer]]
+from_account = "checking"
+from_desc    = "*OUT*"
+to_account   = "checking"
+to_desc      = "*IN*"
+`)
+	writeFile(t, filepath.Join(dir, "out.csv"), "date,description,amount\n2026-02-01,MOVE OUT,-5.00\n")
+	writeFile(t, filepath.Join(dir, "in.csv"), "date,description,amount\n2026-02-02,MOVE IN,5.00\n")
+	call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
+
+	var res status
+	call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "in.csv"}, http.StatusOK, &res)
+	if !strings.Contains(res.Status, "1 transfer leg(s) unpaired") {
+		t.Errorf("status = %q, want the remaining leg reported unpaired", res.Status)
+	}
+}
+
+// A file already gone from disk is forgotten by the index; a name the index
+// never had is not found.
+func TestRemoveFileForgetsAMissingOne(t *testing.T) {
+	dir, h := newUploadServer(t, checkingTOML)
+	writeFile(t, filepath.Join(dir, "a.csv"), statement)
+	call(t, h, "POST", "/api/import", importReq{}, http.StatusOK, nil)
+	if err := os.Remove(filepath.Join(dir, "a.csv")); err != nil {
+		t.Fatal(err)
+	}
+
+	var res status
+	call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "a.csv"}, http.StatusOK, &res)
+	if !strings.HasPrefix(res.Status, "forgot checking/a.csv, 2 transaction(s) dropped") {
+		t.Errorf("status = %q", res.Status)
+	}
+	if got := descriptions(t, h); got != "" {
+		t.Errorf("index still holds %s", got)
+	}
+
+	call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "a.csv"}, http.StatusNotFound, nil)
+	call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: "../rules.toml"}, http.StatusBadRequest, nil)
+	if _, err := os.Stat(filepath.Join(dir, config.AccountFile)); err != nil {
+		t.Fatal(err)
+	}
+	call(t, h, "POST", "/api/files/delete", removeFileReq{Account: "checking", Name: config.AccountFile}, http.StatusNotFound, nil)
+}