Add money: statement-driven personal finance tracker
A data directory holds one folder per account. Statements dropped into those folders are parsed into a rebuildable SQLite index, categorised by ordered glob rules in rules.toml, and browsed or hand-tagged in a Bubble Tea TUI. Movements between the user's own accounts are marked as transfers by the same rules and excluded from spending totals. Manual tags and transfer marks are stored separately from the rule-derived ones and always win, so editing rules.toml and re-running retag never destroys hand edits. Parsers are pluggable. Three are ported from the Python extractors they replace -- nlb and traderepublic read PDFs via pdftotext -layout, revolut reads the CSV export -- alongside a configurable-column CSV parser and a cmd parser that shells out to an external script. Both ports fix two latent bugs in the originals: the sign character class rejected the typographic minus U+2212 that some PDF fonts emit, and NLB's hardcoded continuation indent broke when pdftotext compressed runs of spaces, so the threshold is now measured from the description column. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,283 @@
|
||||
// Package importer walks the data root, extracts transactions from every
|
||||
// statement file, and inserts the ones not already in the index.
|
||||
package importer
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
"git.petrovv.com/nikola/money/internal/glob"
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
"git.petrovv.com/nikola/money/internal/parser"
|
||||
"git.petrovv.com/nikola/money/internal/rules"
|
||||
"git.petrovv.com/nikola/money/internal/store"
|
||||
)
|
||||
|
||||
// FileResult is what happened to one statement file.
|
||||
type FileResult struct {
|
||||
Account string
|
||||
Path string // relative to the data root
|
||||
Parsed int
|
||||
New int
|
||||
Skipped int // already present, i.e. deduplicated
|
||||
Err error
|
||||
// Warnings are non-fatal notes: rows the parser deliberately dropped, and
|
||||
// breaks in the statement's balance chain.
|
||||
Warnings []string
|
||||
}
|
||||
|
||||
// Result summarises a whole import run.
|
||||
type Result struct {
|
||||
Files []FileResult
|
||||
Retagged int
|
||||
}
|
||||
|
||||
// Total counts new rows across every file.
|
||||
func (r Result) Total() (parsed, added, skipped int) {
|
||||
for _, f := range r.Files {
|
||||
parsed += f.Parsed
|
||||
added += f.New
|
||||
skipped += f.Skipped
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Errs collects per-file failures. One unreadable statement must not abort the
|
||||
// whole run, so errors are reported rather than returned.
|
||||
func (r Result) Errs() []FileResult {
|
||||
var out []FileResult
|
||||
for _, f := range r.Files {
|
||||
if f.Err != nil {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Options tunes an import run.
|
||||
type Options struct {
|
||||
// Force reimports statement files whose contents are unchanged. Without
|
||||
// it, files with a matching checksum are skipped entirely.
|
||||
Force bool
|
||||
}
|
||||
|
||||
// Run imports every account under root into db and then applies the rules.
|
||||
func Run(root string, db *store.DB, accounts []*config.Account, engine *rules.Engine, opts Options) (Result, error) {
|
||||
var res Result
|
||||
for _, acc := range accounts {
|
||||
accountID, err := db.UpsertAccount(model.Account{
|
||||
Slug: acc.Slug,
|
||||
Name: acc.Name,
|
||||
Currency: acc.Currency,
|
||||
MinorDigits: acc.Digits(),
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
p, err := parser.For(acc)
|
||||
if err != nil {
|
||||
// A misconfigured account is worth reporting, but the other
|
||||
// accounts should still import.
|
||||
res.Files = append(res.Files, FileResult{Account: acc.Slug, Path: acc.Slug, Err: err})
|
||||
continue
|
||||
}
|
||||
|
||||
files, err := statementFiles(acc)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
for _, path := range files {
|
||||
fr := importFile(root, db, acc, accountID, p, engine, path, opts)
|
||||
res.Files = append(res.Files, fr)
|
||||
}
|
||||
}
|
||||
|
||||
n, err := engine.Retag(db)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Retagged = n
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func importFile(root string, db *store.DB, acc *config.Account, accountID int64,
|
||||
p parser.Parser, engine *rules.Engine, path string, opts Options) FileResult {
|
||||
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
rel = path
|
||||
}
|
||||
fr := FileResult{Account: acc.Slug, Path: rel}
|
||||
|
||||
sum, err := checksum(path)
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
if !opts.Force {
|
||||
prev, seen, err := db.SourceFileSHA(accountID, rel)
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
if seen && prev == sum {
|
||||
return fr // unchanged since last import; nothing to do
|
||||
}
|
||||
}
|
||||
|
||||
txns, err := p.Parse(path, acc)
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
fr.Parsed = len(txns)
|
||||
|
||||
if w, ok := p.(parser.Warner); ok {
|
||||
fr.Warnings = append(fr.Warnings, w.Warnings()...)
|
||||
}
|
||||
fr.Warnings = append(fr.Warnings, checkBalances(txns, acc.Digits())...)
|
||||
|
||||
sourceID, err := db.SourceFile(accountID, rel, sum, time.Now().UTC().Format(time.RFC3339))
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
|
||||
// Identical lines within one statement (two coffees on the same day) are
|
||||
// distinguished by their ordinal, so both survive; the same line seen
|
||||
// again in an overlapping statement gets the same fingerprint and is
|
||||
// deduplicated.
|
||||
seen := map[string]int{}
|
||||
for _, t := range txns {
|
||||
key := fingerprintKey(t)
|
||||
ordinal := seen[key]
|
||||
seen[key]++
|
||||
|
||||
txn := model.Transaction{
|
||||
AccountID: accountID,
|
||||
SourceFileID: sourceID,
|
||||
Fingerprint: fingerprint(key, ordinal),
|
||||
Date: t.Date,
|
||||
Description: t.Description,
|
||||
AmountMinor: t.AmountMinor,
|
||||
Counterparty: t.Counterparty,
|
||||
Type: t.Type,
|
||||
BalanceMinor: t.BalanceMinor,
|
||||
}
|
||||
txn.RuleTag, txn.RuleTransfer = engine.ApplyTxn(acc.Slug, txn)
|
||||
added, err := db.InsertTransaction(txn)
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
if added {
|
||||
fr.New++
|
||||
} else {
|
||||
fr.Skipped++
|
||||
}
|
||||
}
|
||||
return fr
|
||||
}
|
||||
|
||||
// checkBalances verifies that each reported balance is the previous one plus
|
||||
// the transaction amount, which is what the original extraction scripts did.
|
||||
// A break means a row was missed or misparsed, so it is worth saying out loud
|
||||
// even though the import still proceeds.
|
||||
func checkBalances(txns []parser.RawTxn, digits int) []string {
|
||||
var warnings []string
|
||||
var prev *parser.RawTxn
|
||||
|
||||
for i := range txns {
|
||||
t := &txns[i]
|
||||
if t.BalanceMinor == nil {
|
||||
continue // this statement does not report running balances
|
||||
}
|
||||
if prev != nil {
|
||||
expected := *prev.BalanceMinor + t.AmountMinor
|
||||
if expected != *t.BalanceMinor {
|
||||
warnings = append(warnings, fmt.Sprintf(
|
||||
"balance chain breaks at %s %q: statement says %s, previous balance plus amount is %s",
|
||||
t.Date, truncate(t.Description, 40),
|
||||
model.FormatMinor(*t.BalanceMinor, digits),
|
||||
model.FormatMinor(expected, digits)))
|
||||
}
|
||||
}
|
||||
prev = t
|
||||
}
|
||||
return warnings
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
|
||||
func fingerprintKey(t parser.RawTxn) string {
|
||||
return strings.Join([]string{
|
||||
t.Date,
|
||||
fmt.Sprintf("%d", t.AmountMinor),
|
||||
model.NormalizeDescription(t.Description),
|
||||
}, "\x00")
|
||||
}
|
||||
|
||||
func fingerprint(key string, ordinal int) string {
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("%s\x00%d", key, ordinal)))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// statementFiles lists the files in an account folder that should be parsed:
|
||||
// every regular file except account.toml and dotfiles, narrowed by the
|
||||
// account's optional include globs.
|
||||
func statementFiles(acc *config.Account) ([]string, error) {
|
||||
entries, err := os.ReadDir(acc.Dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read account dir %s: %w", acc.Dir, err)
|
||||
}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if e.IsDir() || strings.HasPrefix(name, ".") || name == config.AccountFile {
|
||||
continue
|
||||
}
|
||||
if len(acc.Include) > 0 && !matchAny(acc.Include, name) {
|
||||
continue
|
||||
}
|
||||
out = append(out, filepath.Join(acc.Dir, name))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func matchAny(patterns []string, name string) bool {
|
||||
for _, p := range patterns {
|
||||
if glob.Match(p, name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func checksum(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
"git.petrovv.com/nikola/money/internal/parser"
|
||||
"git.petrovv.com/nikola/money/internal/rules"
|
||||
"git.petrovv.com/nikola/money/internal/store"
|
||||
)
|
||||
|
||||
const accountTOML = `
|
||||
name = "Checking"
|
||||
currency = "EUR"
|
||||
parser = "csv"
|
||||
[csv]
|
||||
skip_rows = 1
|
||||
date = { col = 0, layout = "2006-01-02" }
|
||||
description = { col = 1 }
|
||||
amount = { col = 2 }
|
||||
`
|
||||
|
||||
// newRoot builds a data root with one account and the given statement files.
|
||||
func newRoot(t *testing.T, statements map[string]string) (string, *store.DB, []*config.Account, *rules.Engine) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "checking")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write(t, filepath.Join(dir, config.AccountFile), accountTOML)
|
||||
for name, body := range statements {
|
||||
write(t, filepath.Join(dir, name), body)
|
||||
}
|
||||
|
||||
db, err := store.Open(config.IndexPath(root))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
|
||||
accounts, err := config.LoadAccounts(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := rules.New(&config.Rules{Rule: []config.Rule{{Match: "*LIDL*", Tag: "groceries"}}})
|
||||
return root, db, accounts, engine
|
||||
}
|
||||
|
||||
func write(t *testing.T, path, body string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRun(t *testing.T, root string, db *store.DB, accounts []*config.Account, e *rules.Engine, opts Options) Result {
|
||||
t.Helper()
|
||||
res, err := Run(root, db, accounts, e, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range res.Errs() {
|
||||
t.Fatalf("import of %s failed: %v", f.Path, f.Err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Identical lines inside one statement are distinct transactions; the same
|
||||
// line seen again in an overlapping statement is not.
|
||||
func TestDedupe(t *testing.T) {
|
||||
root, db, accounts, engine := newRoot(t, map[string]string{
|
||||
"2026-01.csv": `date,description,amount
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
2026-01-10,RENT,-800.00
|
||||
`,
|
||||
})
|
||||
|
||||
res := mustRun(t, root, db, accounts, engine, Options{})
|
||||
if _, added, _ := res.Total(); added != 3 {
|
||||
t.Fatalf("first import added %d rows, want 3 (identical same-day lines must both survive)", added)
|
||||
}
|
||||
|
||||
// Unchanged file: skipped without even parsing.
|
||||
res = mustRun(t, root, db, accounts, engine, Options{})
|
||||
if parsed, added, _ := res.Total(); parsed != 0 || added != 0 {
|
||||
t.Errorf("re-import parsed %d and added %d, want 0 and 0", parsed, added)
|
||||
}
|
||||
|
||||
// Same file, parsed again: every row is recognised as a duplicate.
|
||||
res = mustRun(t, root, db, accounts, engine, Options{Force: true})
|
||||
if _, added, skipped := res.Total(); added != 0 || skipped != 3 {
|
||||
t.Errorf("forced re-import added %d, skipped %d; want 0 and 3", added, skipped)
|
||||
}
|
||||
|
||||
// An overlapping statement contributes only its genuinely new rows.
|
||||
write(t, filepath.Join(root, "checking", "2026-02.csv"), `date,description,amount
|
||||
2026-01-10,RENT,-800.00
|
||||
2026-02-10,RENT,-800.00
|
||||
`)
|
||||
accounts, err := config.LoadAccounts(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res = mustRun(t, root, db, accounts, engine, Options{})
|
||||
if _, added, skipped := res.Total(); added != 1 || skipped != 1 {
|
||||
t.Errorf("overlapping import added %d, skipped %d; want 1 and 1", added, skipped)
|
||||
}
|
||||
|
||||
txns, err := db.Transactions(store.Filter{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 4 {
|
||||
t.Errorf("index holds %d transactions, want 4", len(txns))
|
||||
}
|
||||
}
|
||||
|
||||
// Rules are applied as rows are inserted, so a fresh import is already tagged.
|
||||
func TestImportAppliesRules(t *testing.T) {
|
||||
root, db, accounts, engine := newRoot(t, map[string]string{
|
||||
"2026-01.csv": `date,description,amount
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
2026-01-10,RENT,-800.00
|
||||
`,
|
||||
})
|
||||
res := mustRun(t, root, db, accounts, engine, Options{})
|
||||
if res.Retagged != 0 {
|
||||
t.Errorf("retag changed %d rows after import, want 0: rules should already be applied", res.Retagged)
|
||||
}
|
||||
|
||||
untagged, err := db.Transactions(store.Filter{Untagged: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(untagged) != 1 || untagged[0].Description != "RENT" {
|
||||
t.Errorf("untagged = %+v, want only RENT", untagged)
|
||||
}
|
||||
}
|
||||
|
||||
// A broken statement must be reported without aborting the rest of the run.
|
||||
func TestBadFileIsReportedNotFatal(t *testing.T) {
|
||||
root, db, accounts, engine := newRoot(t, map[string]string{
|
||||
"good.csv": `date,description,amount
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
`,
|
||||
"bad.csv": `date,description,amount
|
||||
not-a-date,BROKEN,-1.00
|
||||
`,
|
||||
})
|
||||
|
||||
res, err := Run(root, db, accounts, engine, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Run returned a fatal error, want a per-file report: %v", err)
|
||||
}
|
||||
failures := res.Errs()
|
||||
if len(failures) != 1 || filepath.Base(failures[0].Path) != "bad.csv" {
|
||||
t.Fatalf("failures = %+v, want exactly bad.csv", failures)
|
||||
}
|
||||
if _, added, _ := res.Total(); added != 1 {
|
||||
t.Errorf("added %d rows, want 1 from good.csv", added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckBalances(t *testing.T) {
|
||||
balance := func(v int64) *int64 { return &v }
|
||||
|
||||
good := []parser.RawTxn{
|
||||
{Date: "2026-01-01", Description: "A", AmountMinor: -1000, BalanceMinor: balance(9000)},
|
||||
{Date: "2026-01-02", Description: "B", AmountMinor: -500, BalanceMinor: balance(8500)},
|
||||
{Date: "2026-01-03", Description: "C", AmountMinor: 2000, BalanceMinor: balance(10500)},
|
||||
}
|
||||
if w := checkBalances(good, 2); len(w) != 0 {
|
||||
t.Errorf("a consistent chain produced warnings: %v", w)
|
||||
}
|
||||
|
||||
// A missed row shows up as a break at the row after it.
|
||||
broken := []parser.RawTxn{
|
||||
{Date: "2026-01-01", Description: "A", AmountMinor: -1000, BalanceMinor: balance(9000)},
|
||||
{Date: "2026-01-02", Description: "B", AmountMinor: -500, BalanceMinor: balance(7000)},
|
||||
}
|
||||
w := checkBalances(broken, 2)
|
||||
if len(w) != 1 {
|
||||
t.Fatalf("got %d warnings, want 1: %v", len(w), w)
|
||||
}
|
||||
if !strings.Contains(w[0], "70.00") || !strings.Contains(w[0], "85.00") {
|
||||
t.Errorf("warning = %q, want both the reported and the expected balance", w[0])
|
||||
}
|
||||
|
||||
// Statements that report no balances are not checked.
|
||||
none := []parser.RawTxn{
|
||||
{Date: "2026-01-01", Description: "A", AmountMinor: -1000},
|
||||
{Date: "2026-01-02", Description: "B", AmountMinor: -500},
|
||||
}
|
||||
if w := checkBalances(none, 2); len(w) != 0 {
|
||||
t.Errorf("statements without balances produced warnings: %v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// Only files matching include globs are treated as statements.
|
||||
func TestIncludeGlobs(t *testing.T) {
|
||||
root, db, accounts, engine := newRoot(t, map[string]string{
|
||||
"2026-01.csv": `date,description,amount
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
`,
|
||||
"notes.txt": "not a statement",
|
||||
})
|
||||
accounts[0].Include = []string{"*.csv"}
|
||||
|
||||
res := mustRun(t, root, db, accounts, engine, Options{})
|
||||
if len(res.Files) != 1 {
|
||||
t.Fatalf("processed %d files, want 1: %+v", len(res.Files), res.Files)
|
||||
}
|
||||
if _, added, _ := res.Total(); added != 1 {
|
||||
t.Errorf("added %d rows, want 1", added)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user