Upload statements from the web app

The Accounts screen gets an Add statements panel: pick an account, drop
files on it or choose them, and they are saved into that account's folder
and imported. The folder stays the source of truth -- an upload only puts
a file where `money import` looks, then runs the same import as the
Import button, so deleting index.db and re-importing still loses nothing.

Files travel base64 inside JSON rather than as multipart. There is no
auth, and the JSON-only rule is what keeps another site's form from
posting here; multipart is exactly what such a form can send.

An upload never replaces a statement: identical contents are a no-op and
different ones are refused with 409. Names import would not read back --
not a plain file name, dotfiles, account.toml, outside the account's
include patterns -- are refused, and a batch is checked whole before any
of it is written. Files are written through a dotfile and renamed, so a
concurrent import never reads half of one.

The overview now lists the account folders on disk, read fresh so one
created after startup is a valid target; it replaces the configured
count the empty accounts screen used.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-02 18:51:56 +02:00
co-authored by Claude Opus 5.5
parent ec8a844441
commit 7ec9976b80
6 changed files with 489 additions and 22 deletions
+175 -14
View File
@@ -6,12 +6,15 @@
package web
import (
"bytes"
"embed"
"encoding/json"
"errors"
"fmt"
"io/fs"
"net/http"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
@@ -84,6 +87,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("POST /api/transfers/delete", s.write(s.deleteTransfers))
mux.HandleFunc("POST /api/import", s.write(s.runImport))
mux.HandleFunc("POST /api/upload", s.write(s.upload))
mux.HandleFunc("POST /api/retag", s.write(s.retag))
return mux
}
@@ -159,7 +163,11 @@ func writeJSON(w http.ResponseWriter, code int, v any) {
}
func decode(r *http.Request, v any) error {
dec := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 1<<20))
return decodeLimit(r, v, 1<<20)
}
func decodeLimit(r *http.Request, v any, limit int64) error {
dec := json.NewDecoder(http.MaxBytesReader(nil, r.Body, limit))
dec.DisallowUnknownFields()
if err := dec.Decode(v); err != nil {
return badRequest("bad request body: %v", err)
@@ -167,8 +175,8 @@ func decode(r *http.Request, v any) error {
return nil
}
// status is the reply to every write: one line saying what happened, as the
// TUI's status bar would.
// status is the reply to every write: one line saying what happened, shown
// in the page's status banner.
type status struct {
Status string `json:"status"`
}
@@ -195,14 +203,18 @@ type accountRow struct {
}
type overview struct {
Root string `json:"root"`
// Configured is how many account folders are on disk, which the empty
// accounts screen needs: an account.toml is not enough until an import.
Configured int `json:"configured"`
Accounts []accountRow `json:"accounts"`
Root string `json:"root"`
Accounts []accountRow `json:"accounts"`
// AccountSlugs and Tags are what the builders' fields complete against.
AccountSlugs []string `json:"accountSlugs"`
Tags []string `json:"tags"`
// Folders are the account folders on disk now — what an upload can go
// into, and what the empty accounts screen counts, since an account.toml
// is not enough to appear until an import. Read fresh, since a folder made
// after startup is a valid target.
// A broken account.toml is reported rather than failing the whole page.
Folders []string `json:"folders"`
FoldersErr string `json:"foldersError,omitempty"`
// Stale reports that rules.toml on disk no longer says what the index was
// derived from — it was edited by hand — so the page can offer a retag
// instead of quietly describing rules that are not the ones in force.
@@ -215,7 +227,7 @@ func (s *Server) overview(*http.Request) (any, error) {
if err != nil {
return nil, err
}
out := overview{Root: s.root, Configured: len(s.accounts), Accounts: []accountRow{}}
out := overview{Root: s.root, Accounts: []accountRow{}}
for _, a := range accounts {
bal, err := s.db.Balance(a.ID)
if err != nil {
@@ -236,6 +248,14 @@ func (s *Server) overview(*http.Request) (any, error) {
if out.Tags, err = s.knownTags(); err != nil {
return nil, err
}
out.Folders = []string{}
if folders, err := config.LoadAccounts(s.root); err != nil {
out.FoldersErr = err.Error()
} else {
for _, f := range folders {
out.Folders = append(out.Folders, f.Slug)
}
}
onDisk, err := config.LoadRules(s.root)
if err != nil {
out.Stale, out.RulesErr = true, err.Error()
@@ -1119,22 +1139,26 @@ func (s *Server) runImport(r *http.Request) (any, error) {
if err := decode(r, &req); err != nil {
return nil, err
}
return s.importAll(req.Force)
}
func (s *Server) importAll(force bool) (importJSON, error) {
accounts, err := config.LoadAccounts(s.root)
if err != nil {
return nil, err
return importJSON{}, err
}
if len(accounts) == 0 {
return nil, badRequest("no accounts found in %s (an account is a folder containing %s)",
return importJSON{}, badRequest("no accounts found in %s (an account is a folder containing %s)",
s.root, config.AccountFile)
}
if err := s.reloadRules(); err != nil {
return nil, err
return importJSON{}, err
}
s.accounts = accounts
res, err := importer.Run(s.root, s.db, s.accounts, s.engine, s.links, importer.Options{Force: req.Force})
res, err := importer.Run(s.root, s.db, s.accounts, s.engine, s.links, importer.Options{Force: force})
if err != nil {
return nil, err
return importJSON{}, err
}
_, added, skipped := res.Total()
@@ -1158,6 +1182,143 @@ func (s *Server) runImport(r *http.Request) (any, error) {
return out, nil
}
// uploadLimit caps an upload request. Statements are small — a year of PDF is
// a few hundred kilobytes — and the body is base64, a third larger than the
// files it carries.
const uploadLimit = 64 << 20
type uploadFile struct {
Name string `json:"name"`
Data []byte `json:"data"` // base64 on the wire
}
type uploadReq struct {
Account string `json:"account"`
Files []uploadFile `json:"files"`
}
// upload saves statements into an account folder and imports them. The files
// on disk are the source of truth and the index is derived from them, so an
// upload is nothing more than putting a file where `money import` looks; the
// import that follows is the one the Import button runs.
//
// Files arrive base64 inside JSON rather than as multipart: a multipart body
// is exactly what a cross-site form can send, and the JSON-only rule is the
// only thing standing between another page and this server.
func (s *Server) upload(r *http.Request) (any, error) {
var req uploadReq
if err := decodeLimit(r, &req, uploadLimit); err != nil {
return nil, err
}
if len(req.Files) == 0 {
return nil, badRequest("no files to upload")
}
accounts, err := config.LoadAccounts(s.root)
if err != nil {
return nil, err
}
var acc *config.Account
for _, a := range accounts {
if a.Slug == req.Account {
acc = a
}
}
if acc == nil {
return nil, badRequest("no account folder %q in %s (an account is a folder containing %s)",
req.Account, s.root, config.AccountFile)
}
// Everything is checked before anything is written, so a bad file in a
// batch leaves the folder as it was rather than half uploaded.
var write []uploadFile
var same []string
seen := map[string]bool{}
for _, f := range req.Files {
if err := checkStatementName(acc, f.Name); err != nil {
return nil, err
}
if seen[f.Name] {
return nil, badRequest("%s is in the upload twice", f.Name)
}
seen[f.Name] = true
existing, err := os.ReadFile(filepath.Join(acc.Dir, f.Name))
switch {
case err == nil && bytes.Equal(existing, f.Data):
same = append(same, f.Name)
case err == nil:
// A statement is the source of truth for what it already
// imported; replacing it under the same name is not an upload's
// decision to make.
return nil, &apiError{http.StatusConflict, fmt.Sprintf(
"%s/%s already exists with different contents; rename the file or remove the old one first",
acc.Slug, f.Name)}
case !errors.Is(err, fs.ErrNotExist):
return nil, err
default:
write = append(write, f)
}
}
for _, f := range write {
if err := writeStatement(acc.Dir, f); err != nil {
return nil, err
}
}
out, err := s.importAll(false)
if err != nil {
return nil, err
}
msg := fmt.Sprintf("uploaded %d file(s) to %s", len(write), acc.Slug)
if len(same) > 0 {
msg += fmt.Sprintf(" (%d already there)", len(same))
}
out.Status = msg + " · " + out.Status
return out, nil
}
// checkStatementName refuses any name the importer would not read back as a
// statement of this account, and anything that is not a plain file name.
func checkStatementName(acc *config.Account, name string) error {
if name == "" || name != filepath.Base(name) || strings.ContainsAny(name, "/\\\x00") ||
name == "." || name == ".." {
return badRequest("%q is not a plain file name", name)
}
if strings.HasPrefix(name, ".") || name == config.AccountFile {
return badRequest("%s would be ignored by import (dotfiles and %s are not statements)",
name, config.AccountFile)
}
if len(acc.Include) > 0 && !slices.ContainsFunc(acc.Include, func(p string) bool { return glob.Match(p, name) }) {
return badRequest("%s does not match %s's include patterns (%s), so import would ignore it",
name, acc.Slug, strings.Join(acc.Include, ", "))
}
return nil
}
// writeStatement writes through a dotfile and renames it into place, so a
// concurrent `money import` never reads half a statement.
func writeStatement(dir string, f uploadFile) error {
path := filepath.Join(dir, f.Name)
tmp, err := os.CreateTemp(dir, ".upload-*")
if err != nil {
return fmt.Errorf("write %s: %w", path, err)
}
defer os.Remove(tmp.Name()) // a no-op once renamed
if _, err := tmp.Write(f.Data); err != nil {
tmp.Close()
return fmt.Errorf("write %s: %w", path, err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("write %s: %w", path, err)
}
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
return err
}
if err := os.Rename(tmp.Name(), path); err != nil {
return fmt.Errorf("write %s: %w", path, err)
}
return nil
}
// retag is `money retag`: both halves of what rules.toml decides, re-derived
// from the file as it is on disk now.
func (s *Server) retag(*http.Request) (any, error) {
+99 -7
View File
@@ -203,22 +203,31 @@ async function refreshAll() {
// --- import and retag -----------------------------------------------------
async function runImport(force) {
if (state.busy) return;
function runImport(force) {
return importing(() => api('POST', '/api/import', { force: !!force }));
}
// importing runs a request that ends in an import — the Import button, or an
// upload — and reports it the same way. It answers whether the request
// succeeded; per-file failures inside the import still count as success.
async function importing(request) {
if (state.busy) return false;
state.busy = true;
const btn = document.getElementById('import');
btn.disabled = true;
btn.textContent = 'Importing…';
setStatus('importing… parsing PDF statements can take a while');
try {
const res = await api('POST', '/api/import', { force: !!force });
const res = await request();
state.status = res.status;
state.error = res.failed ? `${res.failed} file(s) failed to import` : '';
renderBanners();
showImportDetails(res);
await refreshAll();
return true;
} catch (e) {
setError(e);
return false;
} finally {
state.busy = false;
btn.disabled = false;
@@ -255,17 +264,19 @@ async function retag() {
function mountAccounts(main) {
const body = h('div');
main.append(h('h2', {}, 'Accounts'), body);
const upload = h('div');
main.append(h('h2', {}, 'Accounts'), body, upload);
async function refresh() {
if (!state.overview) await loadOverview();
const o = state.overview;
upload.replaceChildren(uploader(o));
if (!o.accounts.length) {
// An account.toml is not enough on its own: folders reach the index
// only through an import, so say that rather than show nothing.
body.replaceChildren(h('div', { class: 'panel empty' }, o.configured
? `No accounts imported yet.\n\n${o.configured} account folder(s) configured in ${o.root}.\nPress Import to read their statements.`
: `No account folders found in ${o.root}.\n\nAn account is a folder containing an account.toml.\nCreate one, drop statements in, then press Import.`));
body.replaceChildren(h('div', { class: 'panel empty' }, o.folders.length
? `No accounts imported yet.\n\n${o.folders.length} account folder(s) in ${o.root}.\nAdd statements below, or put them in the folders and press Import.`
: `No account folders found in ${o.root}.\n\nAn account is a folder containing an account.toml.\nCreate one, then add statements to it here.`));
return;
}
body.replaceChildren(h('div', { class: 'panel' }, h('table', {},
@@ -282,6 +293,87 @@ function mountAccounts(main) {
return { refresh };
}
// uploader puts statement files into an account folder and imports them. The
// folder is where they belong: the index is rebuilt from it, so a statement
// that lived only in the index would be lost the next time it is deleted.
function uploader(o) {
const panel = h('div', { class: 'panel upload' }, h('h3', {}, 'Add statements'));
if (!o.folders.length) {
panel.append(h('div', { class: 'muted' }, o.foldersError
? `Account folders do not load: ${o.foldersError}`
: `There is no account folder to add statements to. Create one in ${o.root} with an account.toml.`));
return panel;
}
let files = [];
const account = h('select', { 'aria-label': 'Account to add statements to' },
o.folders.map((f) => h('option', { value: f }, f)));
// Default to the account being browsed, since that is usually the one.
account.value = o.folders.includes(state.filter.account) ? state.filter.account : o.folders[0];
const picker = h('input', {
type: 'file', multiple: true, hidden: true,
onchange: (e) => choose(e.target.files),
});
const chosen = h('div', { class: 'chosen muted' }, 'No files chosen');
const send = h('button', { type: 'button', class: 'primary', disabled: true, onclick: submit }, 'Upload and import');
const zone = h('div', {
class: 'dropzone',
tabindex: 0,
role: 'button',
onclick: () => picker.click(),
onkeydown: (e) => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); picker.click(); } },
ondragover: (e) => { e.preventDefault(); zone.classList.add('over'); },
ondragleave: () => zone.classList.remove('over'),
ondrop: (e) => { e.preventDefault(); zone.classList.remove('over'); choose(e.dataTransfer.files); },
}, 'Drop statement files here, or click to choose');
function choose(list) {
files = Array.from(list || []);
send.disabled = !files.length;
chosen.textContent = files.length
? files.map((f) => `${f.name} (${Math.ceil(f.size / 1024)} KB)`).join(' · ')
: 'No files chosen';
}
async function submit() {
send.disabled = true;
try {
const payload = {
account: account.value,
files: await Promise.all(files.map(async (f) => ({ name: f.name, data: await base64(f) }))),
};
if (await importing(() => api('POST', '/api/upload', payload))) {
picker.value = '';
choose([]);
}
} catch (e) {
setError(e);
} finally {
send.disabled = !files.length;
}
}
panel.append(
h('div', { class: 'toolbar' }, h('label', {}, 'Account ', account), send),
zone, picker, chosen,
h('div', { class: 'hint muted' },
'Files are saved into the account folder, next to the statements already there, and imported. ' +
'A file with the same name and different contents is refused rather than replaced.'));
return panel;
}
// base64 encodes a file for the JSON body. Chunked, because spreading a whole
// statement into String.fromCharCode overflows the call stack.
async function base64(file) {
const bytes = new Uint8Array(await file.arrayBuffer());
let bin = '';
for (let i = 0; i < bytes.length; i += 0x8000) {
bin += String.fromCharCode.apply(null, bytes.subarray(i, i + 0x8000));
}
return btoa(bin);
}
// --- shared scope toolbar ---------------------------------------------------
function scopeToolbar(onChange) {
+20
View File
@@ -176,6 +176,26 @@ td.actions button { padding: 1px 8px; font-size: 12px; }
.kept { font-size: 12px; color: var(--muted); font-family: var(--mono); }
.preview { max-height: calc(100vh - 170px); }
.upload { margin-top: 16px; padding: 14px; overflow: visible; }
.upload h3 { font-size: 14px; margin: 0 0 10px; }
.upload .toolbar { margin-bottom: 10px; }
.dropzone {
border: 2px dashed var(--border);
border-radius: 8px;
padding: 22px 16px;
text-align: center;
color: var(--muted);
cursor: pointer;
}
.dropzone:hover, .dropzone:focus, .dropzone.over {
border-color: var(--accent);
color: var(--accent);
background: var(--accent-soft);
outline: none;
}
.upload .chosen { margin-top: 8px; font-family: var(--mono); font-size: 12px; overflow-wrap: anywhere; }
.upload .hint { margin-top: 8px; font-size: 12px; }
.import-files { margin-top: 6px; font-size: 13px; }
.import-files li { font-family: var(--mono); }
+166
View File
@@ -0,0 +1,166 @@
package web
import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.petrovv.com/nikola/money/internal/config"
"git.petrovv.com/nikola/money/internal/parser"
"git.petrovv.com/nikola/money/internal/rules"
"git.petrovv.com/nikola/money/internal/store"
"git.petrovv.com/nikola/money/internal/transfers"
)
// The upload tests are about where files land and what is refused, not about
// any bank's layout, so they read "date,description,amount" with a header.
func init() {
parser.Register("webtest", func(acc *config.Account) (parser.Parser, error) {
return csvParser{digits: acc.Digits()}, nil
})
}
type csvParser struct{ digits int }
func (p csvParser) Parse(path string, _ *config.Account) ([]parser.RawTxn, error) {
body, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var txns []parser.RawTxn
for i, line := range strings.Split(strings.TrimSpace(string(body)), "\n")[1:] {
f := strings.Split(line, ",")
if len(f) != 3 {
return nil, fmt.Errorf("row %d: want 3 fields", i+2)
}
amount, err := parser.ParseAmount(f[2], ".", "", p.digits)
if err != nil {
return nil, err
}
txns = append(txns, parser.RawTxn{Date: f[0], Description: f[1], AmountMinor: amount})
}
return txns, nil
}
// newUploadServer builds a server over a data root with one empty account
// folder, as `money serve` sees it before the first import.
func newUploadServer(t *testing.T, accountTOML string) (string, http.Handler) {
t.Helper()
root := t.TempDir()
dir := filepath.Join(root, "checking")
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, config.AccountFile), []byte(accountTOML), 0o644); err != nil {
t.Fatal(err)
}
db, err := store.Open(config.IndexPath(root))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Close() })
accounts, err := config.LoadAccounts(root)
if err != nil {
t.Fatal(err)
}
loaded, err := config.LoadRules(root)
if err != nil {
t.Fatal(err)
}
s := New(root, db, accounts, rules.New(loaded), transfers.New(loaded))
s.now = func() time.Time { return time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC) }
return dir, s.Handler()
}
const checkingTOML = "currency = \"EUR\"\nparser = \"webtest\"\n"
const statement = "date,description,amount\n2026-02-01,LIDL SOFIA,-12.50\n2026-02-03,SALARY,1000.00\n"
func TestUploadSavesAndImports(t *testing.T) {
dir, h := newUploadServer(t, checkingTOML)
req := uploadReq{Account: "checking", Files: []uploadFile{{Name: "2026-02.csv", Data: []byte(statement)}}}
var res importJSON
call(t, h, "POST", "/api/upload", req, http.StatusOK, &res)
if !strings.Contains(res.Status, "uploaded 1 file(s) to checking") || !strings.Contains(res.Status, "2 new") {
t.Errorf("status = %q", res.Status)
}
if got, err := os.ReadFile(filepath.Join(dir, "2026-02.csv")); err != nil || string(got) != statement {
t.Errorf("file on disk = %q, %v", got, err)
}
var txns struct{ Rows []txnRow }
call(t, h, "GET", "/api/transactions", nil, http.StatusOK, &txns)
if len(txns.Rows) != 2 {
t.Errorf("index holds %d rows, want the 2 uploaded", len(txns.Rows))
}
// The same file again is not an error, and adds nothing.
call(t, h, "POST", "/api/upload", req, http.StatusOK, &res)
if !strings.Contains(res.Status, "uploaded 0 file(s)") || !strings.Contains(res.Status, "1 already there") {
t.Errorf("re-upload status = %q", res.Status)
}
call(t, h, "GET", "/api/transactions", nil, http.StatusOK, &txns)
if len(txns.Rows) != 2 {
t.Errorf("re-upload left %d rows, want 2", len(txns.Rows))
}
}
// A statement already in the folder is the source of truth for what it
// imported, so an upload never replaces one with different contents.
func TestUploadNeverReplacesAStatement(t *testing.T) {
dir, h := newUploadServer(t, checkingTOML)
path := filepath.Join(dir, "2026-02.csv")
if err := os.WriteFile(path, []byte(statement), 0o644); err != nil {
t.Fatal(err)
}
req := uploadReq{Account: "checking", Files: []uploadFile{{Name: "2026-02.csv", Data: []byte("date,description,amount\n")}}}
call(t, h, "POST", "/api/upload", req, http.StatusConflict, nil)
if got, _ := os.ReadFile(path); string(got) != statement {
t.Errorf("statement was overwritten: %q", got)
}
}
// A batch is checked as a whole before anything is written, and nothing may
// land outside the account folder or where import would not read it back.
func TestUploadRefusesBadNames(t *testing.T) {
dir, h := newUploadServer(t, checkingTOML+"include = [\"*.csv\"]\n")
for _, bad := range []string{"../escape.csv", "sub/x.csv", ".hidden.csv", config.AccountFile, "notes.txt", ""} {
req := uploadReq{Account: "checking", Files: []uploadFile{
{Name: "good.csv", Data: []byte(statement)},
{Name: bad, Data: []byte(statement)},
}}
call(t, h, "POST", "/api/upload", req, http.StatusBadRequest, nil)
}
entries, _ := os.ReadDir(dir)
if len(entries) != 1 {
var names []string
for _, e := range entries {
names = append(names, e.Name())
}
t.Errorf("folder holds %v, want only %s", names, config.AccountFile)
}
if _, err := os.Stat(filepath.Join(filepath.Dir(dir), "escape.csv")); err == nil {
t.Error("a file escaped the account folder")
}
call(t, h, "POST", "/api/upload", uploadReq{Account: "nope", Files: []uploadFile{{Name: "a.csv"}}},
http.StatusBadRequest, nil)
}
// Multipart is what a cross-site form can send, so it is refused like any
// other non-JSON write.
func TestUploadRefusesMultipart(t *testing.T) {
_, h := newUploadServer(t, checkingTOML)
r := httptest.NewRequest("POST", "/api/upload", strings.NewReader("--x\r\n"))
r.Header.Set("Content-Type", "multipart/form-data; boundary=x")
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
if w.Code != http.StatusUnsupportedMediaType {
t.Errorf("status %d, want %d", w.Code, http.StatusUnsupportedMediaType)
}
}