nikolaandClaude Opus 5.5
3d72a54be1
Screen 8 lists every distinct tag -- the ones the index holds and the ones rules.toml names -- with how many transactions carry it and the rules that write it, each by file position with the transactions it wins. It is where near-duplicate tags and tags several rules feed show up, and a rule beaten by a more specific one reads (0) there as it does on the rules screen. A tag no rule names any more says the next retag clears it. The transfer label is not a tag and is never listed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1764 lines
54 KiB
Go
1764 lines
54 KiB
Go
// Package web is money's interactive frontend: a JSON API over the index
|
||
// and rules.toml, and a single page that drives it. It holds no logic of its
|
||
// own about money — every number is formatted, every glob matched and every
|
||
// pair decided by the same packages the CLI uses, so the browser never does
|
||
// arithmetic on an amount or re-implements the matcher.
|
||
package web
|
||
|
||
import (
|
||
"bytes"
|
||
"embed"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io/fs"
|
||
"mime"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"slices"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"git.petrovv.com/nikola/money/internal/config"
|
||
"git.petrovv.com/nikola/money/internal/glob"
|
||
"git.petrovv.com/nikola/money/internal/importer"
|
||
"git.petrovv.com/nikola/money/internal/model"
|
||
"git.petrovv.com/nikola/money/internal/parser"
|
||
"git.petrovv.com/nikola/money/internal/report"
|
||
"git.petrovv.com/nikola/money/internal/rules"
|
||
"git.petrovv.com/nikola/money/internal/store"
|
||
"git.petrovv.com/nikola/money/internal/transfers"
|
||
)
|
||
|
||
//go:embed static
|
||
var static embed.FS
|
||
|
||
// Server holds the index and the engines derived from rules.toml.
|
||
//
|
||
// It outlives any one edit of rules.toml by hand, so retag and
|
||
// import re-read that file (and import the account folders) before running,
|
||
// exactly as a fresh `money retag` or `money import` would. Between those, the
|
||
// engines are what the index was last derived from, which is what every screen
|
||
// must describe; Overview.Stale says when the file on disk has moved on.
|
||
type Server struct {
|
||
root string
|
||
now func() time.Time
|
||
|
||
// mu serialises writers — anything touching rules.toml or the index —
|
||
// against everything else. Readers share it, so browsing stays concurrent
|
||
// while an import holds the index.
|
||
mu sync.RWMutex
|
||
db *store.DB
|
||
accounts []*config.Account
|
||
engine *rules.Engine
|
||
links *transfers.Engine
|
||
}
|
||
|
||
// New builds a server over an opened index and the config loaded from root.
|
||
func New(root string, db *store.DB, accounts []*config.Account, engine *rules.Engine,
|
||
links *transfers.Engine) *Server {
|
||
return &Server{root: root, now: time.Now, db: db, accounts: accounts, engine: engine, links: links}
|
||
}
|
||
|
||
// Handler routes the API and the page.
|
||
func (s *Server) Handler() http.Handler {
|
||
mux := http.NewServeMux()
|
||
page, err := fs.Sub(static, "static")
|
||
if err != nil {
|
||
panic(err) // the embed directive guarantees the directory exists
|
||
}
|
||
mux.Handle("GET /", http.FileServerFS(page))
|
||
|
||
mux.HandleFunc("GET /api/overview", s.read(s.overview))
|
||
mux.HandleFunc("GET /api/transactions", s.read(s.transactions))
|
||
mux.HandleFunc("GET /api/report", s.read(s.report))
|
||
|
||
mux.HandleFunc("GET /api/rules", s.read(s.ruleList))
|
||
mux.HandleFunc("GET /api/tags", s.read(s.tagList))
|
||
mux.HandleFunc("POST /api/rules/preview", s.read(s.rulePreview))
|
||
mux.HandleFunc("POST /api/rules", s.write(s.createRule))
|
||
mux.HandleFunc("PUT /api/rules/{pos}", s.write(s.editRule))
|
||
mux.HandleFunc("POST /api/rules/delete", s.write(s.deleteRules))
|
||
|
||
mux.HandleFunc("GET /api/transfers", s.read(s.transferList))
|
||
mux.HandleFunc("POST /api/transfers/preview", s.read(s.transferPreview))
|
||
mux.HandleFunc("POST /api/transfers", s.write(s.createTransfer))
|
||
mux.HandleFunc("POST /api/transfers/delete", s.write(s.deleteTransfers))
|
||
|
||
mux.HandleFunc("POST /api/import", s.write(s.runImport))
|
||
mux.HandleFunc("POST /api/upload", s.write(s.upload))
|
||
mux.HandleFunc("GET /api/files", s.read(s.fileList))
|
||
mux.HandleFunc("GET /api/files/{account}/{name}", s.serveFile)
|
||
mux.HandleFunc("POST /api/files/delete", s.write(s.removeFile))
|
||
mux.HandleFunc("POST /api/retag", s.write(s.retag))
|
||
return mux
|
||
}
|
||
|
||
// apiError is an error with the HTTP status it should be answered with. Any
|
||
// other error is a 500; the message is shown either way, since it names the
|
||
// file or row at fault and that is what makes it actionable.
|
||
type apiError struct {
|
||
code int
|
||
msg string
|
||
}
|
||
|
||
func (e *apiError) Error() string { return e.msg }
|
||
|
||
func badRequest(format string, args ...any) error {
|
||
return &apiError{http.StatusBadRequest, fmt.Sprintf(format, args...)}
|
||
}
|
||
|
||
// staleRules is the answer to an edit aimed at a rule the page saw but the file
|
||
// no longer holds in that position. Positions are what rules.toml is edited by,
|
||
// so acting on one that moved would rewrite or delete a different rule.
|
||
func staleRules() error {
|
||
return &apiError{http.StatusConflict,
|
||
"rules.toml has changed since this page loaded it; reload and try again"}
|
||
}
|
||
|
||
type handler func(r *http.Request) (any, error)
|
||
|
||
func (s *Server) read(h handler) http.HandlerFunc { return s.serve(h, false) }
|
||
func (s *Server) write(h handler) http.HandlerFunc { return s.serve(h, true) }
|
||
|
||
func (s *Server) serve(h handler, exclusive bool) http.HandlerFunc {
|
||
return func(w http.ResponseWriter, r *http.Request) {
|
||
// There is no authentication, so a body is only accepted as JSON: a
|
||
// cross-site form cannot send that without a preflight, which nothing
|
||
// here answers, so another page open in the browser cannot rewrite
|
||
// rules.toml on the user's behalf.
|
||
if r.Method != http.MethodGet &&
|
||
!strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
|
||
writeJSON(w, http.StatusUnsupportedMediaType,
|
||
map[string]string{"error": "requests must be sent as application/json"})
|
||
return
|
||
}
|
||
if exclusive {
|
||
s.mu.Lock()
|
||
} else {
|
||
s.mu.RLock()
|
||
}
|
||
v, err := h(r)
|
||
if exclusive {
|
||
s.mu.Unlock()
|
||
} else {
|
||
s.mu.RUnlock()
|
||
}
|
||
if err != nil {
|
||
code := http.StatusInternalServerError
|
||
var ae *apiError
|
||
if errors.As(err, &ae) {
|
||
code = ae.code
|
||
}
|
||
writeJSON(w, code, map[string]string{"error": err.Error()})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusOK, v)
|
||
}
|
||
}
|
||
|
||
func writeJSON(w http.ResponseWriter, code int, v any) {
|
||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||
w.Header().Set("Cache-Control", "no-store")
|
||
w.WriteHeader(code)
|
||
json.NewEncoder(w).Encode(v)
|
||
}
|
||
|
||
func decode(r *http.Request, v any) error {
|
||
return decodeLimit(r, v, 1<<20)
|
||
}
|
||
|
||
func decodeLimit(r *http.Request, v any, limit int64) error {
|
||
dec := json.NewDecoder(http.MaxBytesReader(nil, r.Body, limit))
|
||
dec.DisallowUnknownFields()
|
||
if err := dec.Decode(v); err != nil {
|
||
return badRequest("bad request body: %v", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// status is the reply to every write: one line saying what happened, shown
|
||
// in the page's status banner.
|
||
type status struct {
|
||
Status string `json:"status"`
|
||
}
|
||
|
||
// money is an amount ready to show. Formatting happens here, never in the
|
||
// browser, so minor units stay integers end to end.
|
||
type money struct {
|
||
Text string `json:"text"`
|
||
Negative bool `json:"negative"`
|
||
}
|
||
|
||
func formatMoney(minor int64, digits int) money {
|
||
return money{Text: model.FormatMinor(minor, digits), Negative: minor < 0}
|
||
}
|
||
|
||
// --- overview -------------------------------------------------------------
|
||
|
||
type accountRow struct {
|
||
Slug string `json:"slug"`
|
||
Name string `json:"name"`
|
||
Balance money `json:"balance"`
|
||
Currency string `json:"currency"`
|
||
Count int `json:"count"`
|
||
}
|
||
|
||
type overview struct {
|
||
Root string `json:"root"`
|
||
Accounts []accountRow `json:"accounts"`
|
||
// AccountSlugs and Tags are what the builders' fields complete against.
|
||
AccountSlugs []string `json:"accountSlugs"`
|
||
Tags []string `json:"tags"`
|
||
// Folders are the account folders on disk now — what an upload can go
|
||
// into, and what the empty accounts screen counts, since an account.toml
|
||
// is not enough to appear until an import. Read fresh, since a folder made
|
||
// after startup is a valid target.
|
||
// A broken account.toml is reported rather than failing the whole page.
|
||
Folders []string `json:"folders"`
|
||
FoldersErr string `json:"foldersError,omitempty"`
|
||
// Stale reports that rules.toml on disk no longer says what the index was
|
||
// derived from — it was edited by hand — so the page can offer a retag
|
||
// instead of quietly describing rules that are not the ones in force.
|
||
Stale bool `json:"stale"`
|
||
RulesErr string `json:"rulesError,omitempty"`
|
||
}
|
||
|
||
func (s *Server) overview(*http.Request) (any, error) {
|
||
accounts, err := s.db.Accounts()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out := overview{Root: s.root, Accounts: []accountRow{}}
|
||
for _, a := range accounts {
|
||
bal, err := s.db.Balance(a.ID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
n, err := s.db.Count(a.ID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out.Accounts = append(out.Accounts, accountRow{
|
||
Slug: a.Slug, Name: a.Name, Balance: formatMoney(bal, a.MinorDigits),
|
||
Currency: a.Currency, Count: n,
|
||
})
|
||
}
|
||
if out.AccountSlugs, err = s.accountSlugs(); err != nil {
|
||
return nil, err
|
||
}
|
||
if out.Tags, err = s.knownTags(); err != nil {
|
||
return nil, err
|
||
}
|
||
out.Folders = []string{}
|
||
if folders, err := config.LoadAccounts(s.root); err != nil {
|
||
out.FoldersErr = err.Error()
|
||
} else {
|
||
for _, f := range folders {
|
||
out.Folders = append(out.Folders, f.Slug)
|
||
}
|
||
}
|
||
onDisk, err := config.LoadRules(s.root)
|
||
if err != nil {
|
||
out.Stale, out.RulesErr = true, err.Error()
|
||
} else {
|
||
out.Stale = !slices.Equal(onDisk.Rule, s.engine.Rules()) ||
|
||
!slices.Equal(onDisk.Transfer, s.links.Transfers())
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// accountSlugs lists every account worth completing: the folders on disk and
|
||
// whatever the index already holds.
|
||
func (s *Server) accountSlugs() ([]string, error) {
|
||
configured := make([]string, 0, len(s.accounts))
|
||
for _, a := range s.accounts {
|
||
configured = append(configured, a.Slug)
|
||
}
|
||
accounts, err := s.db.Accounts()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
imported := make([]string, 0, len(accounts))
|
||
for _, a := range accounts {
|
||
imported = append(imported, a.Slug)
|
||
}
|
||
return sortedSet(configured, imported), nil
|
||
}
|
||
|
||
func (s *Server) knownAccount(slug string) (bool, error) {
|
||
slugs, err := s.accountSlugs()
|
||
return slices.Contains(slugs, slug), err
|
||
}
|
||
|
||
// knownTags is every tag in use plus every tag a rule names, so a tag is
|
||
// completable from the moment a rule mentions it. model.TransferTag is never
|
||
// among them: it is a label, not a tag, and nothing may be built from it.
|
||
func (s *Server) knownTags() ([]string, error) {
|
||
tagged, err := s.db.Tags()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var fromRules []string
|
||
for _, r := range s.engine.Rules() {
|
||
fromRules = append(fromRules, r.Tag)
|
||
}
|
||
return sortedSet(tagged, fromRules), nil
|
||
}
|
||
|
||
func sortedSet(groups ...[]string) []string {
|
||
seen := map[string]bool{}
|
||
out := []string{}
|
||
for _, g := range groups {
|
||
for _, v := range g {
|
||
if v == "" || seen[v] {
|
||
continue
|
||
}
|
||
seen[v] = true
|
||
out = append(out, v)
|
||
}
|
||
}
|
||
sort.Strings(out)
|
||
return out
|
||
}
|
||
|
||
// --- transactions ---------------------------------------------------------
|
||
|
||
// filterFrom reads the scope the transaction list and the report share. The
|
||
// report's period is deliberately not part of it: it narrows the report and
|
||
// nothing else, so it is read by the report handler alone.
|
||
func filterFrom(r *http.Request) store.Filter {
|
||
q := r.URL.Query()
|
||
return store.Filter{
|
||
AccountSlug: q.Get("account"),
|
||
Search: q.Get("search"),
|
||
Untagged: q.Get("untagged") == "1",
|
||
}
|
||
}
|
||
|
||
type txnRow struct {
|
||
Date string `json:"date"`
|
||
Account string `json:"account"`
|
||
Amount money `json:"amount"`
|
||
Currency string `json:"currency"`
|
||
Tag string `json:"tag"`
|
||
// Supplied marks a tag column the tool filled in — model.TransferTag —
|
||
// rather than one a rule wrote, so it can be shown as a label.
|
||
Supplied bool `json:"supplied"`
|
||
Description string `json:"description"`
|
||
Type string `json:"type"`
|
||
Balance string `json:"balance"`
|
||
}
|
||
|
||
func (s *Server) transactions(r *http.Request) (any, error) {
|
||
txns, err := s.db.Transactions(filterFrom(r))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
rows := make([]txnRow, 0, len(txns))
|
||
for _, t := range txns {
|
||
balance := ""
|
||
if t.BalanceMinor != nil {
|
||
balance = model.FormatMinor(*t.BalanceMinor, t.MinorDigits)
|
||
}
|
||
rows = append(rows, txnRow{
|
||
Date: t.Date, Account: t.AccountSlug, Amount: formatMoney(t.AmountMinor, t.MinorDigits),
|
||
Currency: t.Currency, Tag: t.DisplayTag(), Supplied: t.RuleTag == "" && t.IsTransferLeg(),
|
||
Description: t.Description, Type: t.Type, Balance: balance,
|
||
})
|
||
}
|
||
return map[string]any{"rows": rows}, nil
|
||
}
|
||
|
||
// --- report ---------------------------------------------------------------
|
||
|
||
type periodJSON struct {
|
||
Label string `json:"label"`
|
||
Span string `json:"span"`
|
||
Name string `json:"name"` // label and span together, for a title
|
||
// Bounded is false for all time, the one window with nothing to be empty
|
||
// of: an empty report there means the index is empty, not the period.
|
||
Bounded bool `json:"bounded"`
|
||
}
|
||
|
||
type orderJSON struct {
|
||
Name string `json:"name"`
|
||
Label string `json:"label"`
|
||
Column string `json:"column"`
|
||
Ascending bool `json:"ascending"`
|
||
}
|
||
|
||
type tagRow struct {
|
||
Tag string `json:"tag"`
|
||
Currency string `json:"currency"`
|
||
Out money `json:"out"`
|
||
In money `json:"in"`
|
||
Net money `json:"net"`
|
||
Count int `json:"count"`
|
||
}
|
||
|
||
type excludedRow struct {
|
||
Currency string `json:"currency"`
|
||
Out money `json:"out"`
|
||
In money `json:"in"`
|
||
Net money `json:"net"`
|
||
Legs int `json:"legs"`
|
||
// Fee is present only when a tolerant definition let one through. It is
|
||
// reported, never forgiven: the pair left the report with it inside.
|
||
Fee *money `json:"fee,omitempty"`
|
||
FeeNet *money `json:"feeNet,omitempty"`
|
||
Pairs int `json:"pairs"`
|
||
}
|
||
|
||
type reportJSON struct {
|
||
Periods []periodJSON `json:"periods"`
|
||
Period int `json:"period"`
|
||
Orders []orderJSON `json:"orders"`
|
||
Order string `json:"order"`
|
||
Rows []tagRow `json:"rows"`
|
||
Totals []tagRow `json:"totals"`
|
||
Excluded []excludedRow `json:"excluded"`
|
||
IndexEmpty bool `json:"indexEmpty"`
|
||
}
|
||
|
||
// report answers for one period of the shared scope. The axis is built from
|
||
// the whole index, and the period is named by its label rather than its
|
||
// position, so an import that grows the axis keeps the page on the window it
|
||
// was looking at.
|
||
func (s *Server) report(r *http.Request) (any, error) {
|
||
q := r.URL.Query()
|
||
order := report.OrderOut
|
||
if name := q.Get("sort"); name != "" {
|
||
o, err := report.ParseOrder(name)
|
||
if err != nil {
|
||
return nil, badRequest("%v", err)
|
||
}
|
||
order = o
|
||
}
|
||
|
||
months, err := s.db.Months()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
periods := report.Periods(s.now(), months)
|
||
current := report.DefaultIndex(periods)
|
||
if label := q.Get("period"); label != "" {
|
||
for i, p := range periods {
|
||
if p.Label == label {
|
||
current = i
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
f := filterFrom(r)
|
||
f.From, f.To = periods[current].From, periods[current].To
|
||
txns, err := s.db.Transactions(f)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
out := reportJSON{
|
||
Period: current, Order: order.String(), IndexEmpty: len(months) == 0,
|
||
Rows: []tagRow{}, Totals: []tagRow{}, Excluded: []excludedRow{},
|
||
}
|
||
for _, p := range periods {
|
||
out.Periods = append(out.Periods, periodJSON{
|
||
Label: p.Label, Span: p.Span, Name: p.String(), Bounded: p.From != "",
|
||
})
|
||
}
|
||
for _, o := range report.Orders() {
|
||
out.Orders = append(out.Orders, orderJSON{
|
||
Name: o.String(), Label: o.Label(), Column: o.Column(), Ascending: o.Ascending(),
|
||
})
|
||
}
|
||
|
||
rows := report.ByTag(txns, order)
|
||
for _, t := range rows {
|
||
out.Rows = append(out.Rows, tagRow{
|
||
Tag: t.Tag, Currency: t.Currency, Count: t.Count,
|
||
Out: formatMoney(t.Out, t.Digits), In: formatMoney(t.In, t.Digits),
|
||
Net: formatMoney(t.Net(), t.Digits),
|
||
})
|
||
}
|
||
for _, c := range report.Totals(rows) {
|
||
out.Totals = append(out.Totals, tagRow{
|
||
Tag: "TOTAL", Currency: c.Currency,
|
||
Out: formatMoney(c.Out, c.Digits), In: formatMoney(c.In, c.Digits),
|
||
Net: formatMoney(c.Net(), c.Digits),
|
||
})
|
||
}
|
||
// What ByTag held out, or the report silently disagrees with the account
|
||
// balances by the amount moved between accounts and any fee taken on the way.
|
||
for _, x := range report.Excluded(txns) {
|
||
row := excludedRow{
|
||
Currency: x.Currency, Legs: x.Legs, Pairs: x.Pairs,
|
||
Out: formatMoney(x.Out, x.Digits), In: formatMoney(x.In, x.Digits),
|
||
Net: formatMoney(x.In-x.Out, x.Digits),
|
||
}
|
||
if x.Fee != 0 {
|
||
fee, net := formatMoney(x.Fee, x.Digits), formatMoney(-x.Fee, x.Digits)
|
||
row.Fee, row.FeeNet = &fee, &net
|
||
}
|
||
out.Excluded = append(out.Excluded, row)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// --- rules ----------------------------------------------------------------
|
||
|
||
// ruleJSON is config.Rule on the wire. config.Rule carries only TOML tags, and
|
||
// the type is kept separate on purpose so a wire format change can never leak
|
||
// into rules.toml.
|
||
type ruleJSON struct {
|
||
Match string `json:"match"`
|
||
Type string `json:"type"`
|
||
Account string `json:"account"`
|
||
Tag string `json:"tag"`
|
||
Note string `json:"note"`
|
||
}
|
||
|
||
func toRuleJSON(r config.Rule) ruleJSON {
|
||
return ruleJSON{Match: r.Match, Type: r.Type, Account: r.Account, Tag: r.Tag, Note: r.Note}
|
||
}
|
||
|
||
func (r ruleJSON) rule() config.Rule {
|
||
return config.Rule{Match: r.Match, Type: r.Type, Account: r.Account, Tag: r.Tag, Note: r.Note}
|
||
}
|
||
|
||
type ruleRow struct {
|
||
ruleJSON
|
||
// Pos is the file position, 0-based. Anything that reports a rule reports
|
||
// this, since it is what rules.toml is edited and deleted by.
|
||
Pos int `json:"pos"`
|
||
Pattern string `json:"pattern"`
|
||
Usage int `json:"usage"`
|
||
}
|
||
|
||
// ruleList is every rule in file order with how many transactions it wins —
|
||
// not how many its glob could match — so a shadowed rule shows as zero.
|
||
func (s *Server) ruleList(*http.Request) (any, error) {
|
||
txns, err := s.db.Transactions(store.Filter{})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
rs := s.engine.Rules()
|
||
usage := s.engine.Usage(txns)
|
||
rows := make([]ruleRow, 0, len(rs))
|
||
for i, r := range rs {
|
||
rows = append(rows, ruleRow{
|
||
ruleJSON: toRuleJSON(r), Pos: i, Pattern: rulePattern(r), Usage: usage[i],
|
||
})
|
||
}
|
||
return map[string]any{"rules": rows}, nil
|
||
}
|
||
|
||
type tagRule struct {
|
||
Pos int `json:"pos"` // file position, as the rules screen numbers it
|
||
Pattern string `json:"pattern"`
|
||
Account string `json:"account"`
|
||
Usage int `json:"usage"`
|
||
}
|
||
|
||
type tagEntry struct {
|
||
Tag string `json:"tag"`
|
||
Transactions int `json:"transactions"`
|
||
Rules []tagRule `json:"rules"`
|
||
}
|
||
|
||
// tagList is every distinct tag — the ones the index holds and the ones
|
||
// rules.toml names — with how many transactions carry it and the rules that
|
||
// write it. A tag is only ever what a rule wrote, so the transfer label is
|
||
// never among them, and a tag no rule names any more is one a retag will
|
||
// clear.
|
||
func (s *Server) tagList(*http.Request) (any, error) {
|
||
txns, err := s.db.Transactions(store.Filter{})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
counts := map[string]int{}
|
||
for _, t := range txns {
|
||
if t.RuleTag != "" {
|
||
counts[t.RuleTag]++
|
||
}
|
||
}
|
||
rules := map[string][]tagRule{}
|
||
usage := s.engine.Usage(txns)
|
||
for i, r := range s.engine.Rules() {
|
||
rules[r.Tag] = append(rules[r.Tag], tagRule{Pos: i, Pattern: rulePattern(r), Account: r.Account, Usage: usage[i]})
|
||
}
|
||
tags, err := s.knownTags()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
rows := make([]tagEntry, 0, len(tags))
|
||
for _, tag := range tags {
|
||
rs := rules[tag]
|
||
if rs == nil {
|
||
rs = []tagRule{}
|
||
}
|
||
rows = append(rows, tagEntry{Tag: tag, Transactions: counts[tag], Rules: rs})
|
||
}
|
||
return map[string]any{"tags": rows}, nil
|
||
}
|
||
|
||
// rulePattern renders whichever patterns a rule sets, labelled so a type rule
|
||
// is not mistaken for a description one.
|
||
func rulePattern(r config.Rule) string {
|
||
var parts []string
|
||
if r.Match != "" {
|
||
parts = append(parts, r.Match)
|
||
}
|
||
if r.Type != "" {
|
||
parts = append(parts, "type:"+r.Type)
|
||
}
|
||
return strings.Join(parts, " + ")
|
||
}
|
||
|
||
type rulePreviewReq struct {
|
||
Glob string `json:"glob"`
|
||
Account string `json:"account"`
|
||
// Edit is the file position of the rule being edited, or nil for a new one.
|
||
Edit *int `json:"edit"`
|
||
Sort string `json:"sort"` // "name" or "count"
|
||
}
|
||
|
||
type previewRow struct {
|
||
// Marker is "▸" for a match, "−" for a description the edited rule claims
|
||
// now and would let go of, and "" for context while no glob is typed.
|
||
Marker string `json:"marker"`
|
||
Description string `json:"description"`
|
||
Count int `json:"count"`
|
||
}
|
||
|
||
type rulePreviewJSON struct {
|
||
Rows []previewRow `json:"rows"`
|
||
Matches int `json:"matches"`
|
||
Candidates int `json:"candidates"`
|
||
Dropped int `json:"dropped"`
|
||
}
|
||
|
||
// descGroup is one distinct description the builder previews against. A rule
|
||
// is written against a description, not against rows, so the preview groups.
|
||
type descGroup struct {
|
||
Description string
|
||
Accounts map[string]bool
|
||
Count int
|
||
Claimed bool
|
||
}
|
||
|
||
// rulePreview is the rule builder's right-hand panel: what the glob would claim
|
||
// among everything still waiting for a rule, plus — when editing — what the
|
||
// rule claims already, since those are tagged and an untagged-only preview
|
||
// would be empty for a rule that works. Matching runs here rather than in the
|
||
// browser, so it is the very glob.Match and normalisation the engine uses.
|
||
func (s *Server) rulePreview(r *http.Request) (any, error) {
|
||
var req rulePreviewReq
|
||
if err := decode(r, &req); err != nil {
|
||
return nil, err
|
||
}
|
||
if req.Edit != nil && (*req.Edit < 0 || *req.Edit >= len(s.engine.Rules())) {
|
||
return nil, staleRules()
|
||
}
|
||
|
||
groups, err := s.previewGroups(req.Edit)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
sort.Slice(groups, func(i, j int) bool {
|
||
a, b := groups[i], groups[j]
|
||
// Ties fall back to the name, or the list would reshuffle between
|
||
// keystrokes.
|
||
if req.Sort == "count" && a.Count != b.Count {
|
||
return a.Count > b.Count
|
||
}
|
||
x, y := model.NormalizeDescription(a.Description), model.NormalizeDescription(b.Description)
|
||
if x != y {
|
||
return x < y
|
||
}
|
||
return a.Description < b.Description
|
||
})
|
||
|
||
pattern, account := strings.TrimSpace(req.Glob), strings.TrimSpace(req.Account)
|
||
out := rulePreviewJSON{Rows: []previewRow{}}
|
||
for _, g := range groups {
|
||
inAccount := account == "" || g.Accounts[account]
|
||
if inAccount {
|
||
out.Candidates++
|
||
}
|
||
matched := inAccount &&
|
||
(pattern == "" || glob.Match(pattern, model.NormalizeDescription(g.Description)))
|
||
|
||
marker := ""
|
||
switch {
|
||
case matched && pattern != "":
|
||
marker = "▸"
|
||
out.Matches++
|
||
case matched:
|
||
// No glob yet, so the row is context rather than an answer.
|
||
case g.Claimed:
|
||
// Giving a description up is the decision an edit makes, so it stays
|
||
// on screen instead of vanishing with the other non-matches.
|
||
marker = "−"
|
||
out.Dropped++
|
||
default:
|
||
continue
|
||
}
|
||
out.Rows = append(out.Rows, previewRow{Marker: marker, Description: g.Description, Count: g.Count})
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (s *Server) previewGroups(edit *int) ([]descGroup, error) {
|
||
txns, err := s.db.Transactions(store.Filter{Untagged: true})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
claimedFrom := len(txns)
|
||
if edit != nil {
|
||
seen := make(map[int64]bool, len(txns))
|
||
for _, t := range txns {
|
||
seen[t.ID] = true
|
||
}
|
||
all, err := s.db.Transactions(store.Filter{})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for _, t := range all {
|
||
if seen[t.ID] || s.engine.MatchIndex(t.AccountSlug, t) != *edit {
|
||
continue
|
||
}
|
||
txns = append(txns, t)
|
||
}
|
||
}
|
||
|
||
byDesc := map[string]*descGroup{}
|
||
for i, t := range txns {
|
||
key := model.NormalizeDescription(t.Description)
|
||
g, ok := byDesc[key]
|
||
if !ok {
|
||
g = &descGroup{Description: t.Description, Accounts: map[string]bool{}}
|
||
byDesc[key] = g
|
||
}
|
||
g.Accounts[t.AccountSlug] = true
|
||
g.Count++
|
||
g.Claimed = g.Claimed || i >= claimedFrom
|
||
}
|
||
out := make([]descGroup, 0, len(byDesc))
|
||
for _, g := range byDesc {
|
||
out = append(out, *g)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// ruleForm is the builder's four fields. There is no type field: an edit takes
|
||
// the type from the rule on disk, so a pattern the user was never shown is
|
||
// never one they removed.
|
||
type ruleForm struct {
|
||
Match string `json:"match"`
|
||
Account string `json:"account"`
|
||
Tag string `json:"tag"`
|
||
Note string `json:"note"`
|
||
}
|
||
|
||
func (f ruleForm) rule() config.Rule {
|
||
return config.Rule{
|
||
Match: strings.TrimSpace(f.Match),
|
||
Account: strings.TrimSpace(f.Account),
|
||
Tag: strings.TrimSpace(f.Tag),
|
||
Note: strings.TrimSpace(f.Note),
|
||
}
|
||
}
|
||
|
||
func (s *Server) checkRule(r config.Rule) error {
|
||
if r.Match == "" && r.Type == "" {
|
||
return badRequest("enter a glob first, e.g. *LIDL*")
|
||
}
|
||
if r.Tag == "" {
|
||
return badRequest("enter a tag to apply")
|
||
}
|
||
if r.Account != "" {
|
||
known, err := s.knownAccount(r.Account)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !known {
|
||
return badRequest("no account called %q; leave it blank to apply to every account", r.Account)
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *Server) createRule(r *http.Request) (any, error) {
|
||
var form ruleForm
|
||
if err := decode(r, &form); err != nil {
|
||
return nil, err
|
||
}
|
||
rule := form.rule()
|
||
if err := s.checkRule(rule); err != nil {
|
||
return nil, err
|
||
}
|
||
if err := config.AppendRule(s.root, rule); err != nil {
|
||
return nil, err
|
||
}
|
||
n, err := s.retagAfterSave()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return status{fmt.Sprintf("saved rule %s → %s, %d transactions retagged", rule.Match, rule.Tag, n)}, nil
|
||
}
|
||
|
||
type editRuleReq struct {
|
||
Rule ruleForm `json:"rule"`
|
||
// Expect is the rule as the page showed it. The edit goes ahead only if
|
||
// rules.toml still holds exactly that at the position, since the position
|
||
// alone could by now name a different rule.
|
||
Expect ruleJSON `json:"expect"`
|
||
}
|
||
|
||
func (s *Server) editRule(r *http.Request) (any, error) {
|
||
pos, err := strconv.Atoi(r.PathValue("pos"))
|
||
if err != nil {
|
||
return nil, badRequest("bad rule position %q", r.PathValue("pos"))
|
||
}
|
||
var req editRuleReq
|
||
if err := decode(r, &req); err != nil {
|
||
return nil, err
|
||
}
|
||
onDisk, err := s.expectRules([]int{pos}, []ruleJSON{req.Expect})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
rule := req.Rule.rule()
|
||
rule.Type = onDisk[pos].Type
|
||
if err := s.checkRule(rule); err != nil {
|
||
return nil, err
|
||
}
|
||
if err := config.ReplaceRule(s.root, pos, rule); err != nil {
|
||
return nil, err
|
||
}
|
||
n, err := s.retagAfterSave()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return status{fmt.Sprintf("rule %d is now %s → %s, %d transactions retagged",
|
||
pos+1, rulePattern(rule), rule.Tag, n)}, nil
|
||
}
|
||
|
||
// expectRules checks that rules.toml on disk still holds the expected rule at
|
||
// every position, returning what it holds.
|
||
func (s *Server) expectRules(positions []int, expect []ruleJSON) ([]config.Rule, error) {
|
||
loaded, err := config.LoadRules(s.root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if len(positions) != len(expect) {
|
||
return nil, badRequest("%d positions but %d expected rules", len(positions), len(expect))
|
||
}
|
||
for i, pos := range positions {
|
||
if pos < 0 || pos >= len(loaded.Rule) || loaded.Rule[pos] != expect[i].rule() {
|
||
return nil, staleRules()
|
||
}
|
||
}
|
||
return loaded.Rule, nil
|
||
}
|
||
|
||
// retagAfterSave makes a rules.toml edit take effect: what runs is re-read
|
||
// from disk rather than patched in memory, then every tag is re-derived.
|
||
func (s *Server) retagAfterSave() (int, error) {
|
||
if err := s.reloadRules(); err != nil {
|
||
return 0, fmt.Errorf("rules.toml was written, but re-reading it failed: %w", err)
|
||
}
|
||
return s.engine.Retag(s.db)
|
||
}
|
||
|
||
func (s *Server) reloadRules() error {
|
||
loaded, err := config.LoadRules(s.root)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
s.engine = rules.New(loaded)
|
||
s.links = transfers.New(loaded)
|
||
return nil
|
||
}
|
||
|
||
type deleteRulesReq struct {
|
||
Positions []int `json:"positions"`
|
||
Expect []ruleJSON `json:"expect"`
|
||
}
|
||
|
||
// deleteRules serves both the single delete and the prune: the page sends the
|
||
// positions it confirmed, together with what it showed at each.
|
||
func (s *Server) deleteRules(r *http.Request) (any, error) {
|
||
var req deleteRulesReq
|
||
if err := decode(r, &req); err != nil {
|
||
return nil, err
|
||
}
|
||
if len(req.Positions) == 0 {
|
||
return nil, badRequest("no rules to delete")
|
||
}
|
||
if _, err := s.expectRules(req.Positions, req.Expect); err != nil {
|
||
return nil, err
|
||
}
|
||
n, err := config.DeleteRules(s.root, req.Positions)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
retagged, err := s.retagAfterSave()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return status{fmt.Sprintf("deleted %d rule(s), %d transactions retagged", n, retagged)}, nil
|
||
}
|
||
|
||
// --- transfers ------------------------------------------------------------
|
||
|
||
type transferJSON struct {
|
||
FromAccount string `json:"fromAccount"`
|
||
FromDesc string `json:"fromDesc"`
|
||
ToAccount string `json:"toAccount"`
|
||
ToDesc string `json:"toDesc"`
|
||
TolerancePct float64 `json:"tolerancePct"`
|
||
Note string `json:"note"`
|
||
}
|
||
|
||
func toTransferJSON(t config.Transfer) transferJSON {
|
||
return transferJSON{
|
||
FromAccount: t.FromAccount, FromDesc: t.FromDesc, ToAccount: t.ToAccount,
|
||
ToDesc: t.ToDesc, TolerancePct: t.TolerancePct, Note: t.Note,
|
||
}
|
||
}
|
||
|
||
func (t transferJSON) transfer() config.Transfer {
|
||
return config.Transfer{
|
||
FromAccount: t.FromAccount, FromDesc: t.FromDesc, ToAccount: t.ToAccount,
|
||
ToDesc: t.ToDesc, TolerancePct: t.TolerancePct, Note: t.Note,
|
||
}
|
||
}
|
||
|
||
type transferRow struct {
|
||
transferJSON
|
||
Pos int `json:"pos"`
|
||
Tolerance string `json:"tolerance"` // blank for the exact-amount default
|
||
Paired int `json:"paired"`
|
||
Orphaned int `json:"orphaned"`
|
||
// Legs are the transactions behind Orphaned, newest first, so a ⚠ can be
|
||
// opened to see which movement is missing its other side.
|
||
Legs []unpairedLeg `json:"legs"`
|
||
}
|
||
|
||
type unpairedLeg struct {
|
||
Date string `json:"date"`
|
||
Amount string `json:"amount"`
|
||
Movement string `json:"movement"` // "checking → ?" or "? → savings"
|
||
Description string `json:"description"`
|
||
// Missing names the side not found, "arriving" or "leaving", and Want
|
||
// what it would have had to look like to pair.
|
||
Missing string `json:"missing"`
|
||
Want string `json:"want"`
|
||
}
|
||
|
||
func (s *Server) transferList(*http.Request) (any, error) {
|
||
txns, err := s.db.Transactions(store.Filter{})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
accounts, err := s.db.Accounts()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
currency := map[string]string{}
|
||
digits := map[string]int{}
|
||
for _, a := range accounts {
|
||
currency[a.Slug], digits[a.Slug] = a.Currency, a.MinorDigits
|
||
}
|
||
|
||
defs := s.links.Transfers()
|
||
res := s.links.Analyze(txns)
|
||
rows := make([]transferRow, 0, len(defs))
|
||
for i, t := range defs {
|
||
rows = append(rows, transferRow{
|
||
transferJSON: toTransferJSON(t), Pos: i, Tolerance: formatTolerance(t.TolerancePct),
|
||
Paired: res.Paired[i], Orphaned: res.Orphaned[i], Legs: []unpairedLeg{},
|
||
})
|
||
}
|
||
for _, l := range res.Unmatched {
|
||
row := legRow(l)
|
||
w := s.links.Want(l, func(slug string) string { return currency[slug] })
|
||
d, imported := digits[w.Account]
|
||
if !imported {
|
||
d = l.Txn.MinorDigits
|
||
}
|
||
leg := unpairedLeg{
|
||
Date: row.Date, Amount: row.Amount, Movement: row.Movement, Description: row.Description,
|
||
Missing: "arriving", Want: describeWant(w, d),
|
||
}
|
||
if !l.Out {
|
||
leg.Missing = "leaving"
|
||
}
|
||
if !imported {
|
||
// The commonest cause of all, so it is named outright.
|
||
leg.Want += fmt.Sprintf(" — nothing from %s is imported", w.Account)
|
||
}
|
||
rows[l.Def].Legs = append(rows[l.Def].Legs, leg)
|
||
}
|
||
for i := range rows {
|
||
sort.SliceStable(rows[i].Legs, func(a, b int) bool { return rows[i].Legs[a].Date > rows[i].Legs[b].Date })
|
||
}
|
||
return map[string]any{"transfers": rows, "unpaired": len(res.Unmatched)}, nil
|
||
}
|
||
|
||
// describeWant puts transfers.Wanted into the words the transfers screen uses.
|
||
// The amount is signed as the other statement would show it.
|
||
func describeWant(w transfers.Wanted, digits int) string {
|
||
amount := model.FormatMinor(w.Amount, digits)
|
||
if w.Amount > 0 {
|
||
amount = "+" + amount
|
||
}
|
||
switch {
|
||
case w.AnyAmount:
|
||
amount = "any amount (another currency)"
|
||
case w.TolerancePct > 0:
|
||
amount += " ±" + formatTolerance(w.TolerancePct)
|
||
}
|
||
return fmt.Sprintf("%s in %s matching %s, dated %s to %s", amount, w.Account, w.Desc, w.From, w.To)
|
||
}
|
||
|
||
// legRow is an unpaired leg as the builder's preview and the transfers screen
|
||
// both show it: the amount that moved, and the side that has no partner.
|
||
func legRow(l transfers.Leg) transferPreviewRow {
|
||
amount, movement := l.Txn.AmountMinor, "? → "+l.Txn.AccountSlug
|
||
if l.Out {
|
||
amount, movement = -amount, l.Txn.AccountSlug+" → ?"
|
||
}
|
||
return transferPreviewRow{
|
||
Marker: "⚠", Date: l.Txn.Date, Amount: model.FormatMinor(amount, l.Txn.MinorDigits),
|
||
Movement: movement, Description: l.Txn.Description,
|
||
}
|
||
}
|
||
|
||
// formatTolerance leaves the default blank, so the one or two definitions
|
||
// actually pairing on slack are what the column shows.
|
||
func formatTolerance(pct float64) string {
|
||
if pct == 0 {
|
||
return ""
|
||
}
|
||
return strconv.FormatFloat(pct, 'f', -1, 64) + "%"
|
||
}
|
||
|
||
// transferForm is the builder's fields as typed. The tolerance stays text so a
|
||
// half-typed "1." can still preview; saving is where it has to be a number.
|
||
type transferForm struct {
|
||
FromAccount string `json:"fromAccount"`
|
||
FromDesc string `json:"fromDesc"`
|
||
ToAccount string `json:"toAccount"`
|
||
ToDesc string `json:"toDesc"`
|
||
Tolerance string `json:"tolerance"`
|
||
Note string `json:"note"`
|
||
}
|
||
|
||
func (f transferForm) tolerance() (float64, error) {
|
||
v := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(f.Tolerance), "%"))
|
||
if v == "" {
|
||
return 0, nil
|
||
}
|
||
pct, err := strconv.ParseFloat(v, 64)
|
||
if err != nil {
|
||
return 0, badRequest("tolerance %q is not a number", v)
|
||
}
|
||
return pct, nil
|
||
}
|
||
|
||
func (f transferForm) draft() config.Transfer {
|
||
pct, _ := f.tolerance()
|
||
return config.Transfer{
|
||
FromAccount: strings.TrimSpace(f.FromAccount),
|
||
FromDesc: strings.TrimSpace(f.FromDesc),
|
||
ToAccount: strings.TrimSpace(f.ToAccount),
|
||
ToDesc: strings.TrimSpace(f.ToDesc),
|
||
TolerancePct: pct,
|
||
Note: strings.TrimSpace(f.Note),
|
||
}
|
||
}
|
||
|
||
type transferPreviewRow struct {
|
||
Marker string `json:"marker"` // "▸" a pair, "⚠" a leg with no other side
|
||
Date string `json:"date"`
|
||
Amount string `json:"amount"`
|
||
Movement string `json:"movement"`
|
||
Description string `json:"description"`
|
||
}
|
||
|
||
type transferPreviewJSON struct {
|
||
Rows []transferPreviewRow `json:"rows"`
|
||
Pairs int `json:"pairs"`
|
||
Unpaired int `json:"unpaired"`
|
||
Fees string `json:"fees,omitempty"`
|
||
}
|
||
|
||
// transferPreview pairs the draft *after* the definitions on disk, where
|
||
// saving would put it, so it cannot promise legs an earlier one claims first.
|
||
func (s *Server) transferPreview(r *http.Request) (any, error) {
|
||
var form transferForm
|
||
if err := decode(r, &form); err != nil {
|
||
return nil, err
|
||
}
|
||
txns, err := s.db.Transactions(store.Filter{})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defs := append(append([]config.Transfer(nil), s.links.Transfers()...), form.draft())
|
||
res := transfers.New(&config.Rules{Transfer: defs}).Analyze(txns)
|
||
mine := len(defs) - 1
|
||
|
||
out := transferPreviewJSON{Rows: []transferPreviewRow{}, Pairs: res.Paired[mine], Unpaired: res.Orphaned[mine]}
|
||
var fees int64
|
||
feeDigits := 0
|
||
for _, p := range res.Pairs {
|
||
if p.Def != mine {
|
||
continue
|
||
}
|
||
fees += p.Fee()
|
||
feeDigits = p.Out.MinorDigits
|
||
// Both amounts whenever they disagree: across currencies that is the
|
||
// only place the bank's rate shows, within one it is the fee.
|
||
amount := model.FormatMinor(-p.Out.AmountMinor, p.Out.MinorDigits)
|
||
if p.In.Currency != p.Out.Currency || p.Fee() != 0 {
|
||
amount += " → " + model.FormatMinor(p.In.AmountMinor, p.In.MinorDigits)
|
||
}
|
||
out.Rows = append(out.Rows, transferPreviewRow{
|
||
Marker: "▸", Date: p.Out.Date, Amount: amount,
|
||
Movement: p.Out.AccountSlug + " → " + p.In.AccountSlug, Description: p.Out.Description,
|
||
})
|
||
}
|
||
for _, l := range res.Unmatched {
|
||
if l.Def != mine {
|
||
continue
|
||
}
|
||
out.Rows = append(out.Rows, legRow(l))
|
||
}
|
||
sort.SliceStable(out.Rows, func(i, j int) bool { return out.Rows[i].Date > out.Rows[j].Date })
|
||
if fees != 0 {
|
||
out.Fees = model.FormatMinor(fees, feeDigits)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
func (s *Server) createTransfer(r *http.Request) (any, error) {
|
||
var form transferForm
|
||
if err := decode(r, &form); err != nil {
|
||
return nil, err
|
||
}
|
||
t := form.draft()
|
||
switch {
|
||
case t.FromAccount == "":
|
||
return nil, badRequest("name the account the money leaves")
|
||
case t.FromDesc == "":
|
||
return nil, badRequest("enter a glob for the leaving leg, e.g. *TO REVOLUT*")
|
||
case t.ToAccount == "":
|
||
return nil, badRequest("name the account the money arrives in")
|
||
case t.ToDesc == "":
|
||
return nil, badRequest("enter a glob for the arriving leg, e.g. *FROM NLB*")
|
||
}
|
||
for _, slug := range []string{t.FromAccount, t.ToAccount} {
|
||
known, err := s.knownAccount(slug)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !known {
|
||
return nil, badRequest("no account called %q", slug)
|
||
}
|
||
}
|
||
// draft swallowed this so the preview could keep up with typing; saving is
|
||
// where a value that never became a number is refused, not written as 0.
|
||
if _, err := form.tolerance(); err != nil {
|
||
return nil, err
|
||
}
|
||
if err := config.AppendTransfer(s.root, t); err != nil {
|
||
return nil, badRequest("%v", err)
|
||
}
|
||
paired, unpaired, err := s.relinkAfterSave()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return status{fmt.Sprintf("saved transfer %s → %s, %d matched, %d leg(s) unpaired",
|
||
t.FromAccount, t.ToAccount, paired, unpaired)}, nil
|
||
}
|
||
|
||
func (s *Server) relinkAfterSave() (paired, unpaired int, err error) {
|
||
if err := s.reloadRules(); err != nil {
|
||
return 0, 0, fmt.Errorf("rules.toml was written, but re-reading it failed: %w", err)
|
||
}
|
||
return s.links.Link(s.db)
|
||
}
|
||
|
||
type deleteTransfersReq struct {
|
||
Positions []int `json:"positions"`
|
||
Expect []transferJSON `json:"expect"`
|
||
}
|
||
|
||
func (s *Server) deleteTransfers(r *http.Request) (any, error) {
|
||
var req deleteTransfersReq
|
||
if err := decode(r, &req); err != nil {
|
||
return nil, err
|
||
}
|
||
if len(req.Positions) == 0 {
|
||
return nil, badRequest("no transfers to delete")
|
||
}
|
||
if len(req.Positions) != len(req.Expect) {
|
||
return nil, badRequest("%d positions but %d expected transfers", len(req.Positions), len(req.Expect))
|
||
}
|
||
loaded, err := config.LoadRules(s.root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for i, pos := range req.Positions {
|
||
if pos < 0 || pos >= len(loaded.Transfer) || loaded.Transfer[pos] != req.Expect[i].transfer() {
|
||
return nil, staleRules()
|
||
}
|
||
}
|
||
n, err := config.DeleteTransfers(s.root, req.Positions)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
paired, unpaired, err := s.relinkAfterSave()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return status{fmt.Sprintf("deleted %d transfer(s), %d matched, %d leg(s) unpaired", n, paired, unpaired)}, nil
|
||
}
|
||
|
||
// --- import and retag -----------------------------------------------------
|
||
|
||
type importReq struct {
|
||
Force bool `json:"force"`
|
||
}
|
||
|
||
type importFile struct {
|
||
Path string `json:"path"`
|
||
Parsed int `json:"parsed"`
|
||
New int `json:"new"`
|
||
Skipped int `json:"skipped"`
|
||
Error string `json:"error,omitempty"`
|
||
Warnings []string `json:"warnings,omitempty"`
|
||
}
|
||
|
||
type importJSON struct {
|
||
Status string `json:"status"`
|
||
Files []importFile `json:"files"`
|
||
Failed int `json:"failed"`
|
||
}
|
||
|
||
// runImport is `money import`. The account folders and rules.toml are re-read
|
||
// first, as a fresh process would: the server outlives edits to both, and an
|
||
// import that tagged with yesterday's rules would hand back a stale index.
|
||
// Per-file failures are reported and the rest of the run continues.
|
||
func (s *Server) runImport(r *http.Request) (any, error) {
|
||
var req importReq
|
||
if err := decode(r, &req); err != nil {
|
||
return nil, err
|
||
}
|
||
accounts, err := config.LoadAccounts(s.root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if len(accounts) == 0 {
|
||
return nil, badRequest("no accounts found in %s (an account is a folder containing %s)",
|
||
s.root, config.AccountFile)
|
||
}
|
||
if err := s.reloadRules(); err != nil {
|
||
return nil, err
|
||
}
|
||
s.accounts = accounts
|
||
|
||
res, err := importer.Run(s.root, s.db, s.accounts, s.engine, s.links, importer.Options{Force: req.Force})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
_, added, skipped := res.Total()
|
||
out := importJSON{Files: []importFile{}}
|
||
out.Status = fmt.Sprintf("imported: %d new, %d duplicate", added, skipped)
|
||
if res.Unpaired > 0 {
|
||
out.Status += fmt.Sprintf(" · %d transfer leg(s) unpaired", res.Unpaired)
|
||
}
|
||
for _, f := range res.Files {
|
||
// A file skipped by checksum has nothing to say.
|
||
if f.Err == nil && f.Parsed == 0 && len(f.Warnings) == 0 {
|
||
continue
|
||
}
|
||
file := importFile{Path: f.Path, Parsed: f.Parsed, New: f.New, Skipped: f.Skipped, Warnings: f.Warnings}
|
||
if f.Err != nil {
|
||
file.Error = f.Err.Error()
|
||
out.Failed++
|
||
}
|
||
out.Files = append(out.Files, file)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// uploadLimit caps an upload request. Statements are small — a year of PDF is
|
||
// a few hundred kilobytes — and the body is base64, a third larger than the
|
||
// files it carries.
|
||
const uploadLimit = 64 << 20
|
||
|
||
type uploadFile struct {
|
||
Name string `json:"name"`
|
||
Data []byte `json:"data"` // base64 on the wire
|
||
}
|
||
|
||
type uploadReq struct {
|
||
Account string `json:"account"`
|
||
Files []uploadFile `json:"files"`
|
||
}
|
||
|
||
// upload saves statements into an account folder. The files on disk are the
|
||
// source of truth and the index is derived from them, so an upload is nothing
|
||
// more than putting a file where `money import` looks. It does not import:
|
||
// that stays the user's call, made with the Import button, so a batch can be
|
||
// put together and checked on the statements list before it is read.
|
||
//
|
||
// Files arrive base64 inside JSON rather than as multipart: a multipart body
|
||
// is exactly what a cross-site form can send, and the JSON-only rule is the
|
||
// only thing standing between another page and this server.
|
||
func (s *Server) upload(r *http.Request) (any, error) {
|
||
var req uploadReq
|
||
if err := decodeLimit(r, &req, uploadLimit); err != nil {
|
||
return nil, err
|
||
}
|
||
if len(req.Files) == 0 {
|
||
return nil, badRequest("no files to upload")
|
||
}
|
||
accounts, err := config.LoadAccounts(s.root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var acc *config.Account
|
||
for _, a := range accounts {
|
||
if a.Slug == req.Account {
|
||
acc = a
|
||
}
|
||
}
|
||
if acc == nil {
|
||
return nil, badRequest("no account folder %q in %s (an account is a folder containing %s)",
|
||
req.Account, s.root, config.AccountFile)
|
||
}
|
||
|
||
p, err := parser.For(acc)
|
||
if err != nil {
|
||
return nil, badRequest("%v", err)
|
||
}
|
||
namer, _ := p.(parser.Namer)
|
||
|
||
// Each file is staged as a dotfile — invisible to import — so a parser
|
||
// that names its statements can read it. Everything is then checked
|
||
// before anything is moved into place, so a bad file in a batch leaves
|
||
// the folder as it was rather than half uploaded.
|
||
type staged struct {
|
||
orig, name, tmp string
|
||
data []byte
|
||
named bool // the parser chose name, from the statement
|
||
}
|
||
var batch []*staged
|
||
defer func() {
|
||
for _, f := range batch {
|
||
if f.tmp != "" {
|
||
os.Remove(f.tmp)
|
||
}
|
||
}
|
||
}()
|
||
for _, f := range req.Files {
|
||
if err := checkPlainName(f.Name); err != nil {
|
||
return nil, err
|
||
}
|
||
tmp, err := stageStatement(acc.Dir, f.Data)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
st := &staged{orig: f.Name, name: f.Name, tmp: tmp, data: f.Data}
|
||
batch = append(batch, st)
|
||
if namer != nil {
|
||
stem, err := namer.StatementName(tmp)
|
||
if err != nil {
|
||
return nil, badRequest("%s: %v", f.Name, err)
|
||
}
|
||
if stem != "" {
|
||
st.name, st.named = stem+strings.ToLower(filepath.Ext(f.Name)), true
|
||
}
|
||
}
|
||
}
|
||
|
||
var write []*staged
|
||
var same, renamed []string
|
||
claimed := map[string][]byte{}
|
||
for _, f := range batch {
|
||
base, n := f.name, 2
|
||
resolve:
|
||
for {
|
||
if err := checkStatementName(acc, f.name); err != nil {
|
||
return nil, err
|
||
}
|
||
existing, inBatch := claimed[f.name]
|
||
var err error
|
||
if !inBatch {
|
||
existing, err = os.ReadFile(filepath.Join(acc.Dir, f.name))
|
||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||
return nil, err
|
||
}
|
||
}
|
||
switch {
|
||
case !inBatch && err != nil: // free
|
||
claimed[f.name] = f.data
|
||
write = append(write, f)
|
||
break resolve
|
||
case bytes.Equal(existing, f.data):
|
||
same = append(same, f.orig)
|
||
break resolve
|
||
case f.named:
|
||
// A name the parser chose can be taken by a different
|
||
// statement of the same date — a reissue — so it is
|
||
// numbered, as rename_izpiski.py did, rather than refused.
|
||
f.name = fmt.Sprintf("%s_%d%s", strings.TrimSuffix(base, filepath.Ext(base)), n, filepath.Ext(base))
|
||
n++
|
||
case inBatch:
|
||
return nil, badRequest("%s is in the upload twice", f.orig)
|
||
default:
|
||
// A statement is the source of truth for what it already
|
||
// imported; replacing it under the same name is not an
|
||
// upload's decision to make.
|
||
return nil, &apiError{http.StatusConflict, fmt.Sprintf(
|
||
"%s/%s already exists with different contents; rename the file or remove the old one first",
|
||
acc.Slug, f.name)}
|
||
}
|
||
}
|
||
}
|
||
for _, f := range write {
|
||
path := filepath.Join(acc.Dir, f.name)
|
||
if err := os.Rename(f.tmp, path); err != nil {
|
||
return nil, fmt.Errorf("write %s: %w", path, err)
|
||
}
|
||
f.tmp = ""
|
||
if f.name != f.orig {
|
||
renamed = append(renamed, f.orig+" → "+f.name)
|
||
}
|
||
}
|
||
|
||
msg := fmt.Sprintf("uploaded %d file(s) to %s", len(write), acc.Slug)
|
||
if len(same) > 0 {
|
||
msg += fmt.Sprintf(" (%d already there)", len(same))
|
||
}
|
||
if len(renamed) > 0 {
|
||
msg += ", named by statement date: " + strings.Join(renamed, ", ")
|
||
}
|
||
if len(write) > 0 {
|
||
msg += " · press Import to read them"
|
||
}
|
||
return status{msg}, nil
|
||
}
|
||
|
||
// checkPlainName refuses anything that is not a plain file name, before the
|
||
// file is so much as staged.
|
||
func checkPlainName(name string) error {
|
||
if name == "" || name != filepath.Base(name) || strings.ContainsAny(name, "/\\\x00") ||
|
||
name == "." || name == ".." {
|
||
return badRequest("%q is not a plain file name", name)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// checkStatementName refuses any name the importer would not read back as a
|
||
// statement of this account, and anything that is not a plain file name.
|
||
func checkStatementName(acc *config.Account, name string) error {
|
||
if err := checkPlainName(name); err != nil {
|
||
return err
|
||
}
|
||
if strings.HasPrefix(name, ".") || name == config.AccountFile {
|
||
return badRequest("%s would be ignored by import (dotfiles and %s are not statements)",
|
||
name, config.AccountFile)
|
||
}
|
||
if len(acc.Include) > 0 && !slices.ContainsFunc(acc.Include, func(p string) bool { return glob.Match(p, name) }) {
|
||
return badRequest("%s does not match %s's include patterns (%s), so import would ignore it",
|
||
name, acc.Slug, strings.Join(acc.Include, ", "))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// stageStatement writes an upload to a dotfile in the account folder. Import
|
||
// skips dotfiles, so it is invisible until renamed into place — and a
|
||
// concurrent `money import` never reads half a statement.
|
||
func stageStatement(dir string, data []byte) (string, error) {
|
||
tmp, err := os.CreateTemp(dir, ".upload-*")
|
||
if err != nil {
|
||
return "", fmt.Errorf("write to %s: %w", dir, err)
|
||
}
|
||
if _, err := tmp.Write(data); err != nil {
|
||
tmp.Close()
|
||
os.Remove(tmp.Name())
|
||
return "", fmt.Errorf("write %s: %w", tmp.Name(), err)
|
||
}
|
||
if err := tmp.Close(); err != nil {
|
||
os.Remove(tmp.Name())
|
||
return "", fmt.Errorf("write %s: %w", tmp.Name(), err)
|
||
}
|
||
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
|
||
os.Remove(tmp.Name())
|
||
return "", err
|
||
}
|
||
return tmp.Name(), nil
|
||
}
|
||
|
||
type fileRow struct {
|
||
Account string `json:"account"`
|
||
Name string `json:"name"`
|
||
Size int64 `json:"size"`
|
||
// Modified is the file's mtime, YYYY-MM-DD HH:MM in local time; blank for
|
||
// a file that is gone from disk.
|
||
Modified string `json:"modified"`
|
||
// Status is "imported", "changed" (on disk differs from what was
|
||
// imported), "new" (not imported yet, or its import failed) or "missing"
|
||
// (imported, then removed from disk; its rows stay in the index until it
|
||
// is rebuilt).
|
||
Status string `json:"status"`
|
||
ImportedAt string `json:"importedAt"`
|
||
// Rows is how many transactions the statement holds; Added how many of
|
||
// them it was the first to bring in, which overlap with another statement
|
||
// can make fewer.
|
||
Rows int `json:"rows"`
|
||
Added int `json:"added"`
|
||
}
|
||
|
||
// fileList is every statement on disk next to what the index recorded about
|
||
// it. The folders are the source of truth, so they decide what is listed, and
|
||
// a file the index remembers but the disk no longer holds is called out: its
|
||
// rows would not survive a rebuild.
|
||
func (s *Server) fileList(*http.Request) (any, error) {
|
||
recorded, err := s.db.SourceFiles()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
byPath := map[string]store.SourceFileInfo{}
|
||
for _, f := range recorded {
|
||
byPath[f.Path] = f
|
||
}
|
||
accounts, err := config.LoadAccounts(s.root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
out := []fileRow{}
|
||
onDisk := map[string]bool{}
|
||
for _, acc := range accounts {
|
||
paths, err := importer.StatementFiles(acc)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for _, path := range paths {
|
||
rel, err := filepath.Rel(s.root, path)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
onDisk[rel] = true
|
||
row := fileRow{Account: acc.Slug, Name: filepath.Base(path), Status: "new"}
|
||
if info, err := os.Stat(path); err == nil {
|
||
row.Size, row.Modified = info.Size(), info.ModTime().Format("2006-01-02 15:04")
|
||
}
|
||
if rec, ok := byPath[rel]; ok {
|
||
row.ImportedAt, row.Rows, row.Added, row.Status = rec.ImportedAt, rec.Rows, rec.Added, "imported"
|
||
if sum, err := importer.Checksum(path); err != nil {
|
||
return nil, err
|
||
} else if sum != rec.SHA256 {
|
||
row.Status = "changed"
|
||
}
|
||
}
|
||
out = append(out, row)
|
||
}
|
||
}
|
||
for _, rec := range recorded {
|
||
if !onDisk[rec.Path] {
|
||
out = append(out, fileRow{
|
||
Account: rec.AccountSlug, Name: filepath.Base(rec.Path), Status: "missing",
|
||
ImportedAt: rec.ImportedAt, Rows: rec.Rows, Added: rec.Added,
|
||
})
|
||
}
|
||
}
|
||
sort.SliceStable(out, func(i, j int) bool {
|
||
if out[i].Account != out[j].Account {
|
||
return out[i].Account < out[j].Account
|
||
}
|
||
return out[i].Name < out[j].Name
|
||
})
|
||
return map[string]any{"files": out}, nil
|
||
}
|
||
|
||
// serveFile opens a statement in the browser. Only a file import would read
|
||
// is served — it is looked up in the account's statement list, never joined
|
||
// onto a path — so nothing else under the data root can be fetched.
|
||
//
|
||
// A statement is whatever the bank sent, and it is served from this origin,
|
||
// where a script could drive the API; so nothing is ever rendered as a page.
|
||
// Text opens inline as text/plain under a sandboxing CSP, and anything that is
|
||
// neither text nor PDF downloads. A PDF opens inline without the sandbox: the
|
||
// browsers' viewers refuse to run under one, and they render it in their own
|
||
// isolated viewer rather than in this origin's DOM.
|
||
func (s *Server) serveFile(w http.ResponseWriter, r *http.Request) {
|
||
s.mu.RLock()
|
||
defer s.mu.RUnlock()
|
||
fail := func(code int, msg string) {
|
||
writeJSON(w, code, map[string]string{"error": msg})
|
||
}
|
||
accounts, err := config.LoadAccounts(s.root)
|
||
if err != nil {
|
||
fail(http.StatusInternalServerError, err.Error())
|
||
return
|
||
}
|
||
account, name := r.PathValue("account"), r.PathValue("name")
|
||
var path string
|
||
for _, acc := range accounts {
|
||
if acc.Slug != account {
|
||
continue
|
||
}
|
||
paths, err := importer.StatementFiles(acc)
|
||
if err != nil {
|
||
fail(http.StatusInternalServerError, err.Error())
|
||
return
|
||
}
|
||
for _, p := range paths {
|
||
if filepath.Base(p) == name {
|
||
path = p
|
||
}
|
||
}
|
||
}
|
||
if path == "" {
|
||
fail(http.StatusNotFound, fmt.Sprintf("no statement %s/%s", account, name))
|
||
return
|
||
}
|
||
|
||
disposition := "attachment"
|
||
switch strings.ToLower(filepath.Ext(name)) {
|
||
case ".pdf":
|
||
w.Header().Set("Content-Type", "application/pdf")
|
||
disposition = "inline"
|
||
case ".csv", ".txt", ".tsv":
|
||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||
w.Header().Set("Content-Security-Policy", "sandbox")
|
||
disposition = "inline"
|
||
default:
|
||
w.Header().Set("Content-Type", "application/octet-stream")
|
||
w.Header().Set("Content-Security-Policy", "sandbox")
|
||
}
|
||
w.Header().Set("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": name}))
|
||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||
w.Header().Set("Cache-Control", "no-store")
|
||
http.ServeFile(w, r, path)
|
||
}
|
||
|
||
type removeFileReq struct {
|
||
Account string `json:"account"`
|
||
Name string `json:"name"`
|
||
}
|
||
|
||
// removeFile takes a statement out of the data: the file is deleted and its
|
||
// transactions leave the index. A file already gone from disk is only
|
||
// forgotten by the index. The deletion is permanent — the page says so before
|
||
// asking — so nothing here keeps a copy.
|
||
//
|
||
// It does not import. Rows the file shared with an overlapping statement went
|
||
// with it, and ForgetSourceFile marks the account's other statements so the
|
||
// next Import re-reads them and restores those rows; until then they show as
|
||
// changed. Pairing is re-derived here, though: it reads no statement, and a
|
||
// leg whose partner just went should not stay counted as a transfer.
|
||
func (s *Server) removeFile(r *http.Request) (any, error) {
|
||
var req removeFileReq
|
||
if err := decode(r, &req); err != nil {
|
||
return nil, err
|
||
}
|
||
if err := checkPlainName(req.Name); err != nil {
|
||
return nil, err
|
||
}
|
||
rel := filepath.Join(req.Account, req.Name)
|
||
|
||
// The file is found in the account's statement list, never by joining the
|
||
// request onto a path, exactly as serveFile does.
|
||
var path string
|
||
accounts, err := config.LoadAccounts(s.root)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for _, acc := range accounts {
|
||
if acc.Slug != req.Account {
|
||
continue
|
||
}
|
||
paths, err := importer.StatementFiles(acc)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for _, p := range paths {
|
||
if filepath.Base(p) == req.Name {
|
||
path = p
|
||
}
|
||
}
|
||
}
|
||
|
||
if path != "" {
|
||
if err := os.Remove(path); err != nil {
|
||
return nil, fmt.Errorf("remove %s: %w", path, err)
|
||
}
|
||
}
|
||
dropped, err := s.db.ForgetSourceFile(req.Account, rel)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if path == "" && dropped == 0 {
|
||
known, err := s.db.SourceFiles()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if !slices.ContainsFunc(known, func(f store.SourceFileInfo) bool { return f.Path == rel }) {
|
||
return nil, &apiError{http.StatusNotFound, fmt.Sprintf("no statement %s", rel)}
|
||
}
|
||
}
|
||
|
||
_, unpaired, err := s.links.Link(s.db)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
verb := "deleted"
|
||
if path == "" {
|
||
verb = "forgot"
|
||
}
|
||
msg := fmt.Sprintf("%s %s, %d transaction(s) dropped", verb, rel, dropped)
|
||
if unpaired > 0 {
|
||
msg += fmt.Sprintf(" · %d transfer leg(s) unpaired", unpaired)
|
||
}
|
||
if dropped > 0 {
|
||
msg += " · press Import to restore any of them another statement also holds"
|
||
}
|
||
return status{msg}, nil
|
||
}
|
||
|
||
// retag is `money retag`: both halves of what rules.toml decides, re-derived
|
||
// from the file as it is on disk now.
|
||
func (s *Server) retag(*http.Request) (any, error) {
|
||
if err := s.reloadRules(); err != nil {
|
||
return nil, err
|
||
}
|
||
n, err := s.engine.Retag(s.db)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
paired, unpaired, err := s.links.Link(s.db)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
msg := fmt.Sprintf("rules re-applied, %d rows changed, %d transfers matched", n, paired)
|
||
if unpaired > 0 {
|
||
msg += fmt.Sprintf(", %d leg(s) unpaired", unpaired)
|
||
}
|
||
return status{msg}, nil
|
||
}
|