// Package web is money's interactive frontend: a JSON API over the index // and rules.toml, and a single page that drives it. It holds no logic of its // own about money — every number is formatted, every glob matched and every // pair decided by the same packages the CLI uses, so the browser never does // arithmetic on an amount or re-implements the matcher. package web import ( "bytes" "embed" "encoding/json" "errors" "fmt" "io/fs" "mime" "net/http" "os" "path/filepath" "slices" "sort" "strconv" "strings" "sync" "time" "git.petrovv.com/nikola/money/internal/config" "git.petrovv.com/nikola/money/internal/glob" "git.petrovv.com/nikola/money/internal/importer" "git.petrovv.com/nikola/money/internal/model" "git.petrovv.com/nikola/money/internal/report" "git.petrovv.com/nikola/money/internal/rules" "git.petrovv.com/nikola/money/internal/store" "git.petrovv.com/nikola/money/internal/transfers" ) //go:embed static var static embed.FS // Server holds the index and the engines derived from rules.toml. // // It outlives any one edit of rules.toml by hand, so retag and // import re-read that file (and import the account folders) before running, // exactly as a fresh `money retag` or `money import` would. Between those, the // engines are what the index was last derived from, which is what every screen // must describe; Overview.Stale says when the file on disk has moved on. type Server struct { root string now func() time.Time // mu serialises writers — anything touching rules.toml or the index — // against everything else. Readers share it, so browsing stays concurrent // while an import holds the index. mu sync.RWMutex db *store.DB accounts []*config.Account engine *rules.Engine links *transfers.Engine } // New builds a server over an opened index and the config loaded from root. func New(root string, db *store.DB, accounts []*config.Account, engine *rules.Engine, links *transfers.Engine) *Server { return &Server{root: root, now: time.Now, db: db, accounts: accounts, engine: engine, links: links} } // Handler routes the API and the page. func (s *Server) Handler() http.Handler { mux := http.NewServeMux() page, err := fs.Sub(static, "static") if err != nil { panic(err) // the embed directive guarantees the directory exists } mux.Handle("GET /", http.FileServerFS(page)) mux.HandleFunc("GET /api/overview", s.read(s.overview)) mux.HandleFunc("GET /api/transactions", s.read(s.transactions)) mux.HandleFunc("GET /api/report", s.read(s.report)) mux.HandleFunc("GET /api/rules", s.read(s.ruleList)) mux.HandleFunc("POST /api/rules/preview", s.read(s.rulePreview)) mux.HandleFunc("POST /api/rules", s.write(s.createRule)) mux.HandleFunc("PUT /api/rules/{pos}", s.write(s.editRule)) mux.HandleFunc("POST /api/rules/delete", s.write(s.deleteRules)) mux.HandleFunc("GET /api/transfers", s.read(s.transferList)) mux.HandleFunc("POST /api/transfers/preview", s.read(s.transferPreview)) mux.HandleFunc("POST /api/transfers", s.write(s.createTransfer)) mux.HandleFunc("POST /api/transfers/delete", s.write(s.deleteTransfers)) mux.HandleFunc("POST /api/import", s.write(s.runImport)) mux.HandleFunc("POST /api/upload", s.write(s.upload)) mux.HandleFunc("GET /api/files", s.read(s.fileList)) mux.HandleFunc("GET /api/files/{account}/{name}", s.serveFile) mux.HandleFunc("POST /api/retag", s.write(s.retag)) return mux } // apiError is an error with the HTTP status it should be answered with. Any // other error is a 500; the message is shown either way, since it names the // file or row at fault and that is what makes it actionable. type apiError struct { code int msg string } func (e *apiError) Error() string { return e.msg } func badRequest(format string, args ...any) error { return &apiError{http.StatusBadRequest, fmt.Sprintf(format, args...)} } // staleRules is the answer to an edit aimed at a rule the page saw but the file // no longer holds in that position. Positions are what rules.toml is edited by, // so acting on one that moved would rewrite or delete a different rule. func staleRules() error { return &apiError{http.StatusConflict, "rules.toml has changed since this page loaded it; reload and try again"} } type handler func(r *http.Request) (any, error) func (s *Server) read(h handler) http.HandlerFunc { return s.serve(h, false) } func (s *Server) write(h handler) http.HandlerFunc { return s.serve(h, true) } func (s *Server) serve(h handler, exclusive bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { // There is no authentication, so a body is only accepted as JSON: a // cross-site form cannot send that without a preflight, which nothing // here answers, so another page open in the browser cannot rewrite // rules.toml on the user's behalf. if r.Method != http.MethodGet && !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { writeJSON(w, http.StatusUnsupportedMediaType, map[string]string{"error": "requests must be sent as application/json"}) return } if exclusive { s.mu.Lock() } else { s.mu.RLock() } v, err := h(r) if exclusive { s.mu.Unlock() } else { s.mu.RUnlock() } if err != nil { code := http.StatusInternalServerError var ae *apiError if errors.As(err, &ae) { code = ae.code } writeJSON(w, code, map[string]string{"error": err.Error()}) return } writeJSON(w, http.StatusOK, v) } } func writeJSON(w http.ResponseWriter, code int, v any) { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Cache-Control", "no-store") w.WriteHeader(code) json.NewEncoder(w).Encode(v) } func decode(r *http.Request, v any) error { return decodeLimit(r, v, 1<<20) } func decodeLimit(r *http.Request, v any, limit int64) error { dec := json.NewDecoder(http.MaxBytesReader(nil, r.Body, limit)) dec.DisallowUnknownFields() if err := dec.Decode(v); err != nil { return badRequest("bad request body: %v", err) } return nil } // status is the reply to every write: one line saying what happened, shown // in the page's status banner. type status struct { Status string `json:"status"` } // money is an amount ready to show. Formatting happens here, never in the // browser, so minor units stay integers end to end. type money struct { Text string `json:"text"` Negative bool `json:"negative"` } func formatMoney(minor int64, digits int) money { return money{Text: model.FormatMinor(minor, digits), Negative: minor < 0} } // --- overview ------------------------------------------------------------- type accountRow struct { Slug string `json:"slug"` Name string `json:"name"` Balance money `json:"balance"` Currency string `json:"currency"` Count int `json:"count"` } type overview struct { Root string `json:"root"` Accounts []accountRow `json:"accounts"` // AccountSlugs and Tags are what the builders' fields complete against. AccountSlugs []string `json:"accountSlugs"` Tags []string `json:"tags"` // Folders are the account folders on disk now — what an upload can go // into, and what the empty accounts screen counts, since an account.toml // is not enough to appear until an import. Read fresh, since a folder made // after startup is a valid target. // A broken account.toml is reported rather than failing the whole page. Folders []string `json:"folders"` FoldersErr string `json:"foldersError,omitempty"` // Stale reports that rules.toml on disk no longer says what the index was // derived from — it was edited by hand — so the page can offer a retag // instead of quietly describing rules that are not the ones in force. Stale bool `json:"stale"` RulesErr string `json:"rulesError,omitempty"` } func (s *Server) overview(*http.Request) (any, error) { accounts, err := s.db.Accounts() if err != nil { return nil, err } out := overview{Root: s.root, Accounts: []accountRow{}} for _, a := range accounts { bal, err := s.db.Balance(a.ID) if err != nil { return nil, err } n, err := s.db.Count(a.ID) if err != nil { return nil, err } out.Accounts = append(out.Accounts, accountRow{ Slug: a.Slug, Name: a.Name, Balance: formatMoney(bal, a.MinorDigits), Currency: a.Currency, Count: n, }) } if out.AccountSlugs, err = s.accountSlugs(); err != nil { return nil, err } if out.Tags, err = s.knownTags(); err != nil { return nil, err } out.Folders = []string{} if folders, err := config.LoadAccounts(s.root); err != nil { out.FoldersErr = err.Error() } else { for _, f := range folders { out.Folders = append(out.Folders, f.Slug) } } onDisk, err := config.LoadRules(s.root) if err != nil { out.Stale, out.RulesErr = true, err.Error() } else { out.Stale = !slices.Equal(onDisk.Rule, s.engine.Rules()) || !slices.Equal(onDisk.Transfer, s.links.Transfers()) } return out, nil } // accountSlugs lists every account worth completing: the folders on disk and // whatever the index already holds. func (s *Server) accountSlugs() ([]string, error) { configured := make([]string, 0, len(s.accounts)) for _, a := range s.accounts { configured = append(configured, a.Slug) } accounts, err := s.db.Accounts() if err != nil { return nil, err } imported := make([]string, 0, len(accounts)) for _, a := range accounts { imported = append(imported, a.Slug) } return sortedSet(configured, imported), nil } func (s *Server) knownAccount(slug string) (bool, error) { slugs, err := s.accountSlugs() return slices.Contains(slugs, slug), err } // knownTags is every tag in use plus every tag a rule names, so a tag is // completable from the moment a rule mentions it. model.TransferTag is never // among them: it is a label, not a tag, and nothing may be built from it. func (s *Server) knownTags() ([]string, error) { tagged, err := s.db.Tags() if err != nil { return nil, err } var fromRules []string for _, r := range s.engine.Rules() { fromRules = append(fromRules, r.Tag) } return sortedSet(tagged, fromRules), nil } func sortedSet(groups ...[]string) []string { seen := map[string]bool{} out := []string{} for _, g := range groups { for _, v := range g { if v == "" || seen[v] { continue } seen[v] = true out = append(out, v) } } sort.Strings(out) return out } // --- transactions --------------------------------------------------------- // filterFrom reads the scope the transaction list and the report share. The // report's period is deliberately not part of it: it narrows the report and // nothing else, so it is read by the report handler alone. func filterFrom(r *http.Request) store.Filter { q := r.URL.Query() return store.Filter{ AccountSlug: q.Get("account"), Search: q.Get("search"), Untagged: q.Get("untagged") == "1", } } type txnRow struct { Date string `json:"date"` Account string `json:"account"` Amount money `json:"amount"` Currency string `json:"currency"` Tag string `json:"tag"` // Supplied marks a tag column the tool filled in — model.TransferTag — // rather than one a rule wrote, so it can be shown as a label. Supplied bool `json:"supplied"` Description string `json:"description"` Type string `json:"type"` Balance string `json:"balance"` } func (s *Server) transactions(r *http.Request) (any, error) { txns, err := s.db.Transactions(filterFrom(r)) if err != nil { return nil, err } rows := make([]txnRow, 0, len(txns)) for _, t := range txns { balance := "" if t.BalanceMinor != nil { balance = model.FormatMinor(*t.BalanceMinor, t.MinorDigits) } rows = append(rows, txnRow{ Date: t.Date, Account: t.AccountSlug, Amount: formatMoney(t.AmountMinor, t.MinorDigits), Currency: t.Currency, Tag: t.DisplayTag(), Supplied: t.RuleTag == "" && t.IsTransferLeg(), Description: t.Description, Type: t.Type, Balance: balance, }) } return map[string]any{"rows": rows}, nil } // --- report --------------------------------------------------------------- type periodJSON struct { Label string `json:"label"` Span string `json:"span"` Name string `json:"name"` // label and span together, for a title // Bounded is false for all time, the one window with nothing to be empty // of: an empty report there means the index is empty, not the period. Bounded bool `json:"bounded"` } type orderJSON struct { Name string `json:"name"` Label string `json:"label"` Column string `json:"column"` Ascending bool `json:"ascending"` } type tagRow struct { Tag string `json:"tag"` Currency string `json:"currency"` Out money `json:"out"` In money `json:"in"` Net money `json:"net"` Count int `json:"count"` } type excludedRow struct { Currency string `json:"currency"` Out money `json:"out"` In money `json:"in"` Net money `json:"net"` Legs int `json:"legs"` // Fee is present only when a tolerant definition let one through. It is // reported, never forgiven: the pair left the report with it inside. Fee *money `json:"fee,omitempty"` FeeNet *money `json:"feeNet,omitempty"` Pairs int `json:"pairs"` } type reportJSON struct { Periods []periodJSON `json:"periods"` Period int `json:"period"` Orders []orderJSON `json:"orders"` Order string `json:"order"` Rows []tagRow `json:"rows"` Totals []tagRow `json:"totals"` Excluded []excludedRow `json:"excluded"` IndexEmpty bool `json:"indexEmpty"` } // report answers for one period of the shared scope. The axis is built from // the whole index, and the period is named by its label rather than its // position, so an import that grows the axis keeps the page on the window it // was looking at. func (s *Server) report(r *http.Request) (any, error) { q := r.URL.Query() order := report.OrderOut if name := q.Get("sort"); name != "" { o, err := report.ParseOrder(name) if err != nil { return nil, badRequest("%v", err) } order = o } months, err := s.db.Months() if err != nil { return nil, err } periods := report.Periods(s.now(), months) current := report.DefaultIndex(periods) if label := q.Get("period"); label != "" { for i, p := range periods { if p.Label == label { current = i break } } } f := filterFrom(r) f.From, f.To = periods[current].From, periods[current].To txns, err := s.db.Transactions(f) if err != nil { return nil, err } out := reportJSON{ Period: current, Order: order.String(), IndexEmpty: len(months) == 0, Rows: []tagRow{}, Totals: []tagRow{}, Excluded: []excludedRow{}, } for _, p := range periods { out.Periods = append(out.Periods, periodJSON{ Label: p.Label, Span: p.Span, Name: p.String(), Bounded: p.From != "", }) } for _, o := range report.Orders() { out.Orders = append(out.Orders, orderJSON{ Name: o.String(), Label: o.Label(), Column: o.Column(), Ascending: o.Ascending(), }) } rows := report.ByTag(txns, order) for _, t := range rows { out.Rows = append(out.Rows, tagRow{ Tag: t.Tag, Currency: t.Currency, Count: t.Count, Out: formatMoney(t.Out, t.Digits), In: formatMoney(t.In, t.Digits), Net: formatMoney(t.Net(), t.Digits), }) } for _, c := range report.Totals(rows) { out.Totals = append(out.Totals, tagRow{ Tag: "TOTAL", Currency: c.Currency, Out: formatMoney(c.Out, c.Digits), In: formatMoney(c.In, c.Digits), Net: formatMoney(c.Net(), c.Digits), }) } // What ByTag held out, or the report silently disagrees with the account // balances by the amount moved between accounts and any fee taken on the way. for _, x := range report.Excluded(txns) { row := excludedRow{ Currency: x.Currency, Legs: x.Legs, Pairs: x.Pairs, Out: formatMoney(x.Out, x.Digits), In: formatMoney(x.In, x.Digits), Net: formatMoney(x.In-x.Out, x.Digits), } if x.Fee != 0 { fee, net := formatMoney(x.Fee, x.Digits), formatMoney(-x.Fee, x.Digits) row.Fee, row.FeeNet = &fee, &net } out.Excluded = append(out.Excluded, row) } return out, nil } // --- rules ---------------------------------------------------------------- // ruleJSON is config.Rule on the wire. config.Rule carries only TOML tags, and // the type is kept separate on purpose so a wire format change can never leak // into rules.toml. type ruleJSON struct { Match string `json:"match"` Type string `json:"type"` Account string `json:"account"` Tag string `json:"tag"` Note string `json:"note"` } func toRuleJSON(r config.Rule) ruleJSON { return ruleJSON{Match: r.Match, Type: r.Type, Account: r.Account, Tag: r.Tag, Note: r.Note} } func (r ruleJSON) rule() config.Rule { return config.Rule{Match: r.Match, Type: r.Type, Account: r.Account, Tag: r.Tag, Note: r.Note} } type ruleRow struct { ruleJSON // Pos is the file position, 0-based. Anything that reports a rule reports // this, since it is what rules.toml is edited and deleted by. Pos int `json:"pos"` Pattern string `json:"pattern"` Usage int `json:"usage"` } // ruleList is every rule in file order with how many transactions it wins — // not how many its glob could match — so a shadowed rule shows as zero. func (s *Server) ruleList(*http.Request) (any, error) { txns, err := s.db.Transactions(store.Filter{}) if err != nil { return nil, err } rs := s.engine.Rules() usage := s.engine.Usage(txns) rows := make([]ruleRow, 0, len(rs)) for i, r := range rs { rows = append(rows, ruleRow{ ruleJSON: toRuleJSON(r), Pos: i, Pattern: rulePattern(r), Usage: usage[i], }) } return map[string]any{"rules": rows}, nil } // rulePattern renders whichever patterns a rule sets, labelled so a type rule // is not mistaken for a description one. func rulePattern(r config.Rule) string { var parts []string if r.Match != "" { parts = append(parts, r.Match) } if r.Type != "" { parts = append(parts, "type:"+r.Type) } return strings.Join(parts, " + ") } type rulePreviewReq struct { Glob string `json:"glob"` Account string `json:"account"` // Edit is the file position of the rule being edited, or nil for a new one. Edit *int `json:"edit"` Sort string `json:"sort"` // "name" or "count" } type previewRow struct { // Marker is "▸" for a match, "−" for a description the edited rule claims // now and would let go of, and "" for context while no glob is typed. Marker string `json:"marker"` Description string `json:"description"` Count int `json:"count"` } type rulePreviewJSON struct { Rows []previewRow `json:"rows"` Matches int `json:"matches"` Candidates int `json:"candidates"` Dropped int `json:"dropped"` } // descGroup is one distinct description the builder previews against. A rule // is written against a description, not against rows, so the preview groups. type descGroup struct { Description string Accounts map[string]bool Count int Claimed bool } // rulePreview is the rule builder's right-hand panel: what the glob would claim // among everything still waiting for a rule, plus — when editing — what the // rule claims already, since those are tagged and an untagged-only preview // would be empty for a rule that works. Matching runs here rather than in the // browser, so it is the very glob.Match and normalisation the engine uses. func (s *Server) rulePreview(r *http.Request) (any, error) { var req rulePreviewReq if err := decode(r, &req); err != nil { return nil, err } if req.Edit != nil && (*req.Edit < 0 || *req.Edit >= len(s.engine.Rules())) { return nil, staleRules() } groups, err := s.previewGroups(req.Edit) if err != nil { return nil, err } sort.Slice(groups, func(i, j int) bool { a, b := groups[i], groups[j] // Ties fall back to the name, or the list would reshuffle between // keystrokes. if req.Sort == "count" && a.Count != b.Count { return a.Count > b.Count } x, y := model.NormalizeDescription(a.Description), model.NormalizeDescription(b.Description) if x != y { return x < y } return a.Description < b.Description }) pattern, account := strings.TrimSpace(req.Glob), strings.TrimSpace(req.Account) out := rulePreviewJSON{Rows: []previewRow{}} for _, g := range groups { inAccount := account == "" || g.Accounts[account] if inAccount { out.Candidates++ } matched := inAccount && (pattern == "" || glob.Match(pattern, model.NormalizeDescription(g.Description))) marker := "" switch { case matched && pattern != "": marker = "▸" out.Matches++ case matched: // No glob yet, so the row is context rather than an answer. case g.Claimed: // Giving a description up is the decision an edit makes, so it stays // on screen instead of vanishing with the other non-matches. marker = "−" out.Dropped++ default: continue } out.Rows = append(out.Rows, previewRow{Marker: marker, Description: g.Description, Count: g.Count}) } return out, nil } func (s *Server) previewGroups(edit *int) ([]descGroup, error) { txns, err := s.db.Transactions(store.Filter{Untagged: true}) if err != nil { return nil, err } claimedFrom := len(txns) if edit != nil { seen := make(map[int64]bool, len(txns)) for _, t := range txns { seen[t.ID] = true } all, err := s.db.Transactions(store.Filter{}) if err != nil { return nil, err } for _, t := range all { if seen[t.ID] || s.engine.MatchIndex(t.AccountSlug, t) != *edit { continue } txns = append(txns, t) } } byDesc := map[string]*descGroup{} for i, t := range txns { key := model.NormalizeDescription(t.Description) g, ok := byDesc[key] if !ok { g = &descGroup{Description: t.Description, Accounts: map[string]bool{}} byDesc[key] = g } g.Accounts[t.AccountSlug] = true g.Count++ g.Claimed = g.Claimed || i >= claimedFrom } out := make([]descGroup, 0, len(byDesc)) for _, g := range byDesc { out = append(out, *g) } return out, nil } // ruleForm is the builder's four fields. There is no type field: an edit takes // the type from the rule on disk, so a pattern the user was never shown is // never one they removed. type ruleForm struct { Match string `json:"match"` Account string `json:"account"` Tag string `json:"tag"` Note string `json:"note"` } func (f ruleForm) rule() config.Rule { return config.Rule{ Match: strings.TrimSpace(f.Match), Account: strings.TrimSpace(f.Account), Tag: strings.TrimSpace(f.Tag), Note: strings.TrimSpace(f.Note), } } func (s *Server) checkRule(r config.Rule) error { if r.Match == "" && r.Type == "" { return badRequest("enter a glob first, e.g. *LIDL*") } if r.Tag == "" { return badRequest("enter a tag to apply") } if r.Account != "" { known, err := s.knownAccount(r.Account) if err != nil { return err } if !known { return badRequest("no account called %q; leave it blank to apply to every account", r.Account) } } return nil } func (s *Server) createRule(r *http.Request) (any, error) { var form ruleForm if err := decode(r, &form); err != nil { return nil, err } rule := form.rule() if err := s.checkRule(rule); err != nil { return nil, err } if err := config.AppendRule(s.root, rule); err != nil { return nil, err } n, err := s.retagAfterSave() if err != nil { return nil, err } return status{fmt.Sprintf("saved rule %s → %s, %d transactions retagged", rule.Match, rule.Tag, n)}, nil } type editRuleReq struct { Rule ruleForm `json:"rule"` // Expect is the rule as the page showed it. The edit goes ahead only if // rules.toml still holds exactly that at the position, since the position // alone could by now name a different rule. Expect ruleJSON `json:"expect"` } func (s *Server) editRule(r *http.Request) (any, error) { pos, err := strconv.Atoi(r.PathValue("pos")) if err != nil { return nil, badRequest("bad rule position %q", r.PathValue("pos")) } var req editRuleReq if err := decode(r, &req); err != nil { return nil, err } onDisk, err := s.expectRules([]int{pos}, []ruleJSON{req.Expect}) if err != nil { return nil, err } rule := req.Rule.rule() rule.Type = onDisk[pos].Type if err := s.checkRule(rule); err != nil { return nil, err } if err := config.ReplaceRule(s.root, pos, rule); err != nil { return nil, err } n, err := s.retagAfterSave() if err != nil { return nil, err } return status{fmt.Sprintf("rule %d is now %s → %s, %d transactions retagged", pos+1, rulePattern(rule), rule.Tag, n)}, nil } // expectRules checks that rules.toml on disk still holds the expected rule at // every position, returning what it holds. func (s *Server) expectRules(positions []int, expect []ruleJSON) ([]config.Rule, error) { loaded, err := config.LoadRules(s.root) if err != nil { return nil, err } if len(positions) != len(expect) { return nil, badRequest("%d positions but %d expected rules", len(positions), len(expect)) } for i, pos := range positions { if pos < 0 || pos >= len(loaded.Rule) || loaded.Rule[pos] != expect[i].rule() { return nil, staleRules() } } return loaded.Rule, nil } // retagAfterSave makes a rules.toml edit take effect: what runs is re-read // from disk rather than patched in memory, then every tag is re-derived. func (s *Server) retagAfterSave() (int, error) { if err := s.reloadRules(); err != nil { return 0, fmt.Errorf("rules.toml was written, but re-reading it failed: %w", err) } return s.engine.Retag(s.db) } func (s *Server) reloadRules() error { loaded, err := config.LoadRules(s.root) if err != nil { return err } s.engine = rules.New(loaded) s.links = transfers.New(loaded) return nil } type deleteRulesReq struct { Positions []int `json:"positions"` Expect []ruleJSON `json:"expect"` } // deleteRules serves both the single delete and the prune: the page sends the // positions it confirmed, together with what it showed at each. func (s *Server) deleteRules(r *http.Request) (any, error) { var req deleteRulesReq if err := decode(r, &req); err != nil { return nil, err } if len(req.Positions) == 0 { return nil, badRequest("no rules to delete") } if _, err := s.expectRules(req.Positions, req.Expect); err != nil { return nil, err } n, err := config.DeleteRules(s.root, req.Positions) if err != nil { return nil, err } retagged, err := s.retagAfterSave() if err != nil { return nil, err } return status{fmt.Sprintf("deleted %d rule(s), %d transactions retagged", n, retagged)}, nil } // --- transfers ------------------------------------------------------------ type transferJSON struct { FromAccount string `json:"fromAccount"` FromDesc string `json:"fromDesc"` ToAccount string `json:"toAccount"` ToDesc string `json:"toDesc"` TolerancePct float64 `json:"tolerancePct"` Note string `json:"note"` } func toTransferJSON(t config.Transfer) transferJSON { return transferJSON{ FromAccount: t.FromAccount, FromDesc: t.FromDesc, ToAccount: t.ToAccount, ToDesc: t.ToDesc, TolerancePct: t.TolerancePct, Note: t.Note, } } func (t transferJSON) transfer() config.Transfer { return config.Transfer{ FromAccount: t.FromAccount, FromDesc: t.FromDesc, ToAccount: t.ToAccount, ToDesc: t.ToDesc, TolerancePct: t.TolerancePct, Note: t.Note, } } type transferRow struct { transferJSON Pos int `json:"pos"` Tolerance string `json:"tolerance"` // blank for the exact-amount default Paired int `json:"paired"` Orphaned int `json:"orphaned"` // Legs are the transactions behind Orphaned, newest first, so a ⚠ can be // opened to see which movement is missing its other side. Legs []unpairedLeg `json:"legs"` } type unpairedLeg struct { Date string `json:"date"` Amount string `json:"amount"` Movement string `json:"movement"` // "checking → ?" or "? → savings" Description string `json:"description"` // Missing names the side not found, "arriving" or "leaving", and Want // what it would have had to look like to pair. Missing string `json:"missing"` Want string `json:"want"` } func (s *Server) transferList(*http.Request) (any, error) { txns, err := s.db.Transactions(store.Filter{}) if err != nil { return nil, err } accounts, err := s.db.Accounts() if err != nil { return nil, err } currency := map[string]string{} digits := map[string]int{} for _, a := range accounts { currency[a.Slug], digits[a.Slug] = a.Currency, a.MinorDigits } defs := s.links.Transfers() res := s.links.Analyze(txns) rows := make([]transferRow, 0, len(defs)) for i, t := range defs { rows = append(rows, transferRow{ transferJSON: toTransferJSON(t), Pos: i, Tolerance: formatTolerance(t.TolerancePct), Paired: res.Paired[i], Orphaned: res.Orphaned[i], Legs: []unpairedLeg{}, }) } for _, l := range res.Unmatched { row := legRow(l) w := s.links.Want(l, func(slug string) string { return currency[slug] }) d, imported := digits[w.Account] if !imported { d = l.Txn.MinorDigits } leg := unpairedLeg{ Date: row.Date, Amount: row.Amount, Movement: row.Movement, Description: row.Description, Missing: "arriving", Want: describeWant(w, d), } if !l.Out { leg.Missing = "leaving" } if !imported { // The commonest cause of all, so it is named outright. leg.Want += fmt.Sprintf(" — nothing from %s is imported", w.Account) } rows[l.Def].Legs = append(rows[l.Def].Legs, leg) } for i := range rows { sort.SliceStable(rows[i].Legs, func(a, b int) bool { return rows[i].Legs[a].Date > rows[i].Legs[b].Date }) } return map[string]any{"transfers": rows, "unpaired": len(res.Unmatched)}, nil } // describeWant puts transfers.Wanted into the words the transfers screen uses. // The amount is signed as the other statement would show it. func describeWant(w transfers.Wanted, digits int) string { amount := model.FormatMinor(w.Amount, digits) if w.Amount > 0 { amount = "+" + amount } switch { case w.AnyAmount: amount = "any amount (another currency)" case w.TolerancePct > 0: amount += " ±" + formatTolerance(w.TolerancePct) } return fmt.Sprintf("%s in %s matching %s, dated %s to %s", amount, w.Account, w.Desc, w.From, w.To) } // legRow is an unpaired leg as the builder's preview and the transfers screen // both show it: the amount that moved, and the side that has no partner. func legRow(l transfers.Leg) transferPreviewRow { amount, movement := l.Txn.AmountMinor, "? → "+l.Txn.AccountSlug if l.Out { amount, movement = -amount, l.Txn.AccountSlug+" → ?" } return transferPreviewRow{ Marker: "⚠", Date: l.Txn.Date, Amount: model.FormatMinor(amount, l.Txn.MinorDigits), Movement: movement, Description: l.Txn.Description, } } // formatTolerance leaves the default blank, so the one or two definitions // actually pairing on slack are what the column shows. func formatTolerance(pct float64) string { if pct == 0 { return "" } return strconv.FormatFloat(pct, 'f', -1, 64) + "%" } // transferForm is the builder's fields as typed. The tolerance stays text so a // half-typed "1." can still preview; saving is where it has to be a number. type transferForm struct { FromAccount string `json:"fromAccount"` FromDesc string `json:"fromDesc"` ToAccount string `json:"toAccount"` ToDesc string `json:"toDesc"` Tolerance string `json:"tolerance"` Note string `json:"note"` } func (f transferForm) tolerance() (float64, error) { v := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(f.Tolerance), "%")) if v == "" { return 0, nil } pct, err := strconv.ParseFloat(v, 64) if err != nil { return 0, badRequest("tolerance %q is not a number", v) } return pct, nil } func (f transferForm) draft() config.Transfer { pct, _ := f.tolerance() return config.Transfer{ FromAccount: strings.TrimSpace(f.FromAccount), FromDesc: strings.TrimSpace(f.FromDesc), ToAccount: strings.TrimSpace(f.ToAccount), ToDesc: strings.TrimSpace(f.ToDesc), TolerancePct: pct, Note: strings.TrimSpace(f.Note), } } type transferPreviewRow struct { Marker string `json:"marker"` // "▸" a pair, "⚠" a leg with no other side Date string `json:"date"` Amount string `json:"amount"` Movement string `json:"movement"` Description string `json:"description"` } type transferPreviewJSON struct { Rows []transferPreviewRow `json:"rows"` Pairs int `json:"pairs"` Unpaired int `json:"unpaired"` Fees string `json:"fees,omitempty"` } // transferPreview pairs the draft *after* the definitions on disk, where // saving would put it, so it cannot promise legs an earlier one claims first. func (s *Server) transferPreview(r *http.Request) (any, error) { var form transferForm if err := decode(r, &form); err != nil { return nil, err } txns, err := s.db.Transactions(store.Filter{}) if err != nil { return nil, err } defs := append(append([]config.Transfer(nil), s.links.Transfers()...), form.draft()) res := transfers.New(&config.Rules{Transfer: defs}).Analyze(txns) mine := len(defs) - 1 out := transferPreviewJSON{Rows: []transferPreviewRow{}, Pairs: res.Paired[mine], Unpaired: res.Orphaned[mine]} var fees int64 feeDigits := 0 for _, p := range res.Pairs { if p.Def != mine { continue } fees += p.Fee() feeDigits = p.Out.MinorDigits // Both amounts whenever they disagree: across currencies that is the // only place the bank's rate shows, within one it is the fee. amount := model.FormatMinor(-p.Out.AmountMinor, p.Out.MinorDigits) if p.In.Currency != p.Out.Currency || p.Fee() != 0 { amount += " → " + model.FormatMinor(p.In.AmountMinor, p.In.MinorDigits) } out.Rows = append(out.Rows, transferPreviewRow{ Marker: "▸", Date: p.Out.Date, Amount: amount, Movement: p.Out.AccountSlug + " → " + p.In.AccountSlug, Description: p.Out.Description, }) } for _, l := range res.Unmatched { if l.Def != mine { continue } out.Rows = append(out.Rows, legRow(l)) } sort.SliceStable(out.Rows, func(i, j int) bool { return out.Rows[i].Date > out.Rows[j].Date }) if fees != 0 { out.Fees = model.FormatMinor(fees, feeDigits) } return out, nil } func (s *Server) createTransfer(r *http.Request) (any, error) { var form transferForm if err := decode(r, &form); err != nil { return nil, err } t := form.draft() switch { case t.FromAccount == "": return nil, badRequest("name the account the money leaves") case t.FromDesc == "": return nil, badRequest("enter a glob for the leaving leg, e.g. *TO REVOLUT*") case t.ToAccount == "": return nil, badRequest("name the account the money arrives in") case t.ToDesc == "": return nil, badRequest("enter a glob for the arriving leg, e.g. *FROM NLB*") } for _, slug := range []string{t.FromAccount, t.ToAccount} { known, err := s.knownAccount(slug) if err != nil { return nil, err } if !known { return nil, badRequest("no account called %q", slug) } } // draft swallowed this so the preview could keep up with typing; saving is // where a value that never became a number is refused, not written as 0. if _, err := form.tolerance(); err != nil { return nil, err } if err := config.AppendTransfer(s.root, t); err != nil { return nil, badRequest("%v", err) } paired, unpaired, err := s.relinkAfterSave() if err != nil { return nil, err } return status{fmt.Sprintf("saved transfer %s → %s, %d matched, %d leg(s) unpaired", t.FromAccount, t.ToAccount, paired, unpaired)}, nil } func (s *Server) relinkAfterSave() (paired, unpaired int, err error) { if err := s.reloadRules(); err != nil { return 0, 0, fmt.Errorf("rules.toml was written, but re-reading it failed: %w", err) } return s.links.Link(s.db) } type deleteTransfersReq struct { Positions []int `json:"positions"` Expect []transferJSON `json:"expect"` } func (s *Server) deleteTransfers(r *http.Request) (any, error) { var req deleteTransfersReq if err := decode(r, &req); err != nil { return nil, err } if len(req.Positions) == 0 { return nil, badRequest("no transfers to delete") } if len(req.Positions) != len(req.Expect) { return nil, badRequest("%d positions but %d expected transfers", len(req.Positions), len(req.Expect)) } loaded, err := config.LoadRules(s.root) if err != nil { return nil, err } for i, pos := range req.Positions { if pos < 0 || pos >= len(loaded.Transfer) || loaded.Transfer[pos] != req.Expect[i].transfer() { return nil, staleRules() } } n, err := config.DeleteTransfers(s.root, req.Positions) if err != nil { return nil, err } paired, unpaired, err := s.relinkAfterSave() if err != nil { return nil, err } return status{fmt.Sprintf("deleted %d transfer(s), %d matched, %d leg(s) unpaired", n, paired, unpaired)}, nil } // --- import and retag ----------------------------------------------------- type importReq struct { Force bool `json:"force"` } type importFile struct { Path string `json:"path"` Parsed int `json:"parsed"` New int `json:"new"` Skipped int `json:"skipped"` Error string `json:"error,omitempty"` Warnings []string `json:"warnings,omitempty"` } type importJSON struct { Status string `json:"status"` Files []importFile `json:"files"` Failed int `json:"failed"` } // runImport is `money import`. The account folders and rules.toml are re-read // first, as a fresh process would: the server outlives edits to both, and an // import that tagged with yesterday's rules would hand back a stale index. // Per-file failures are reported and the rest of the run continues. func (s *Server) runImport(r *http.Request) (any, error) { var req importReq if err := decode(r, &req); err != nil { return nil, err } return s.importAll(req.Force) } func (s *Server) importAll(force bool) (importJSON, error) { accounts, err := config.LoadAccounts(s.root) if err != nil { return importJSON{}, err } if len(accounts) == 0 { return importJSON{}, badRequest("no accounts found in %s (an account is a folder containing %s)", s.root, config.AccountFile) } if err := s.reloadRules(); err != nil { return importJSON{}, err } s.accounts = accounts res, err := importer.Run(s.root, s.db, s.accounts, s.engine, s.links, importer.Options{Force: force}) if err != nil { return importJSON{}, err } _, added, skipped := res.Total() out := importJSON{Files: []importFile{}} out.Status = fmt.Sprintf("imported: %d new, %d duplicate", added, skipped) if res.Unpaired > 0 { out.Status += fmt.Sprintf(" · %d transfer leg(s) unpaired", res.Unpaired) } for _, f := range res.Files { // A file skipped by checksum has nothing to say. if f.Err == nil && f.Parsed == 0 && len(f.Warnings) == 0 { continue } file := importFile{Path: f.Path, Parsed: f.Parsed, New: f.New, Skipped: f.Skipped, Warnings: f.Warnings} if f.Err != nil { file.Error = f.Err.Error() out.Failed++ } out.Files = append(out.Files, file) } return out, nil } // uploadLimit caps an upload request. Statements are small — a year of PDF is // a few hundred kilobytes — and the body is base64, a third larger than the // files it carries. const uploadLimit = 64 << 20 type uploadFile struct { Name string `json:"name"` Data []byte `json:"data"` // base64 on the wire } type uploadReq struct { Account string `json:"account"` Files []uploadFile `json:"files"` } // upload saves statements into an account folder and imports them. The files // on disk are the source of truth and the index is derived from them, so an // upload is nothing more than putting a file where `money import` looks; the // import that follows is the one the Import button runs. // // Files arrive base64 inside JSON rather than as multipart: a multipart body // is exactly what a cross-site form can send, and the JSON-only rule is the // only thing standing between another page and this server. func (s *Server) upload(r *http.Request) (any, error) { var req uploadReq if err := decodeLimit(r, &req, uploadLimit); err != nil { return nil, err } if len(req.Files) == 0 { return nil, badRequest("no files to upload") } accounts, err := config.LoadAccounts(s.root) if err != nil { return nil, err } var acc *config.Account for _, a := range accounts { if a.Slug == req.Account { acc = a } } if acc == nil { return nil, badRequest("no account folder %q in %s (an account is a folder containing %s)", req.Account, s.root, config.AccountFile) } // Everything is checked before anything is written, so a bad file in a // batch leaves the folder as it was rather than half uploaded. var write []uploadFile var same []string seen := map[string]bool{} for _, f := range req.Files { if err := checkStatementName(acc, f.Name); err != nil { return nil, err } if seen[f.Name] { return nil, badRequest("%s is in the upload twice", f.Name) } seen[f.Name] = true existing, err := os.ReadFile(filepath.Join(acc.Dir, f.Name)) switch { case err == nil && bytes.Equal(existing, f.Data): same = append(same, f.Name) case err == nil: // A statement is the source of truth for what it already // imported; replacing it under the same name is not an upload's // decision to make. return nil, &apiError{http.StatusConflict, fmt.Sprintf( "%s/%s already exists with different contents; rename the file or remove the old one first", acc.Slug, f.Name)} case !errors.Is(err, fs.ErrNotExist): return nil, err default: write = append(write, f) } } for _, f := range write { if err := writeStatement(acc.Dir, f); err != nil { return nil, err } } out, err := s.importAll(false) if err != nil { return nil, err } msg := fmt.Sprintf("uploaded %d file(s) to %s", len(write), acc.Slug) if len(same) > 0 { msg += fmt.Sprintf(" (%d already there)", len(same)) } out.Status = msg + " · " + out.Status return out, nil } // checkStatementName refuses any name the importer would not read back as a // statement of this account, and anything that is not a plain file name. func checkStatementName(acc *config.Account, name string) error { if name == "" || name != filepath.Base(name) || strings.ContainsAny(name, "/\\\x00") || name == "." || name == ".." { return badRequest("%q is not a plain file name", name) } if strings.HasPrefix(name, ".") || name == config.AccountFile { return badRequest("%s would be ignored by import (dotfiles and %s are not statements)", name, config.AccountFile) } if len(acc.Include) > 0 && !slices.ContainsFunc(acc.Include, func(p string) bool { return glob.Match(p, name) }) { return badRequest("%s does not match %s's include patterns (%s), so import would ignore it", name, acc.Slug, strings.Join(acc.Include, ", ")) } return nil } // writeStatement writes through a dotfile and renames it into place, so a // concurrent `money import` never reads half a statement. func writeStatement(dir string, f uploadFile) error { path := filepath.Join(dir, f.Name) tmp, err := os.CreateTemp(dir, ".upload-*") if err != nil { return fmt.Errorf("write %s: %w", path, err) } defer os.Remove(tmp.Name()) // a no-op once renamed if _, err := tmp.Write(f.Data); err != nil { tmp.Close() return fmt.Errorf("write %s: %w", path, err) } if err := tmp.Close(); err != nil { return fmt.Errorf("write %s: %w", path, err) } if err := os.Chmod(tmp.Name(), 0o644); err != nil { return err } if err := os.Rename(tmp.Name(), path); err != nil { return fmt.Errorf("write %s: %w", path, err) } return nil } type fileRow struct { Account string `json:"account"` Name string `json:"name"` Size int64 `json:"size"` // Modified is the file's mtime, YYYY-MM-DD HH:MM in local time; blank for // a file that is gone from disk. Modified string `json:"modified"` // Status is "imported", "changed" (on disk differs from what was // imported), "new" (not imported yet, or its import failed) or "missing" // (imported, then removed from disk; its rows stay in the index until it // is rebuilt). Status string `json:"status"` ImportedAt string `json:"importedAt"` Added int `json:"added"` } // fileList is every statement on disk next to what the index recorded about // it. The folders are the source of truth, so they decide what is listed, and // a file the index remembers but the disk no longer holds is called out: its // rows would not survive a rebuild. func (s *Server) fileList(*http.Request) (any, error) { recorded, err := s.db.SourceFiles() if err != nil { return nil, err } byPath := map[string]store.SourceFileInfo{} for _, f := range recorded { byPath[f.Path] = f } accounts, err := config.LoadAccounts(s.root) if err != nil { return nil, err } out := []fileRow{} onDisk := map[string]bool{} for _, acc := range accounts { paths, err := importer.StatementFiles(acc) if err != nil { return nil, err } for _, path := range paths { rel, err := filepath.Rel(s.root, path) if err != nil { return nil, err } onDisk[rel] = true row := fileRow{Account: acc.Slug, Name: filepath.Base(path), Status: "new"} if info, err := os.Stat(path); err == nil { row.Size, row.Modified = info.Size(), info.ModTime().Format("2006-01-02 15:04") } if rec, ok := byPath[rel]; ok { row.ImportedAt, row.Added, row.Status = rec.ImportedAt, rec.Added, "imported" if sum, err := importer.Checksum(path); err != nil { return nil, err } else if sum != rec.SHA256 { row.Status = "changed" } } out = append(out, row) } } for _, rec := range recorded { if !onDisk[rec.Path] { out = append(out, fileRow{ Account: rec.AccountSlug, Name: filepath.Base(rec.Path), Status: "missing", ImportedAt: rec.ImportedAt, Added: rec.Added, }) } } sort.SliceStable(out, func(i, j int) bool { if out[i].Account != out[j].Account { return out[i].Account < out[j].Account } return out[i].Name < out[j].Name }) return map[string]any{"files": out}, nil } // serveFile opens a statement in the browser. Only a file import would read // is served — it is looked up in the account's statement list, never joined // onto a path — so nothing else under the data root can be fetched. // // A statement is whatever the bank sent, and it is served from this origin, // where a script could drive the API; so nothing is ever rendered as a page. // Text opens inline as text/plain under a sandboxing CSP, and anything that is // neither text nor PDF downloads. A PDF opens inline without the sandbox: the // browsers' viewers refuse to run under one, and they render it in their own // isolated viewer rather than in this origin's DOM. func (s *Server) serveFile(w http.ResponseWriter, r *http.Request) { s.mu.RLock() defer s.mu.RUnlock() fail := func(code int, msg string) { writeJSON(w, code, map[string]string{"error": msg}) } accounts, err := config.LoadAccounts(s.root) if err != nil { fail(http.StatusInternalServerError, err.Error()) return } account, name := r.PathValue("account"), r.PathValue("name") var path string for _, acc := range accounts { if acc.Slug != account { continue } paths, err := importer.StatementFiles(acc) if err != nil { fail(http.StatusInternalServerError, err.Error()) return } for _, p := range paths { if filepath.Base(p) == name { path = p } } } if path == "" { fail(http.StatusNotFound, fmt.Sprintf("no statement %s/%s", account, name)) return } disposition := "attachment" switch strings.ToLower(filepath.Ext(name)) { case ".pdf": w.Header().Set("Content-Type", "application/pdf") disposition = "inline" case ".csv", ".txt", ".tsv": w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Content-Security-Policy", "sandbox") disposition = "inline" default: w.Header().Set("Content-Type", "application/octet-stream") w.Header().Set("Content-Security-Policy", "sandbox") } w.Header().Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": name})) w.Header().Set("X-Content-Type-Options", "nosniff") w.Header().Set("Cache-Control", "no-store") http.ServeFile(w, r, path) } // retag is `money retag`: both halves of what rules.toml decides, re-derived // from the file as it is on disk now. func (s *Server) retag(*http.Request) (any, error) { if err := s.reloadRules(); err != nil { return nil, err } n, err := s.engine.Retag(s.db) if err != nil { return nil, err } paired, unpaired, err := s.links.Link(s.db) if err != nil { return nil, err } msg := fmt.Sprintf("rules re-applied, %d rows changed, %d transfers matched", n, paired) if unpaired > 0 { msg += fmt.Sprintf(", %d leg(s) unpaired", unpaired) } return status{msg}, nil }