Add money: statement-driven personal finance tracker
A data directory holds one folder per account. Statements dropped into those folders are parsed into a rebuildable SQLite index, categorised by ordered glob rules in rules.toml, and browsed or hand-tagged in a Bubble Tea TUI. Movements between the user's own accounts are marked as transfers by the same rules and excluded from spending totals. Manual tags and transfer marks are stored separately from the rule-derived ones and always win, so editing rules.toml and re-running retag never destroys hand edits. Parsers are pluggable. Three are ported from the Python extractors they replace -- nlb and traderepublic read PDFs via pdftotext -layout, revolut reads the CSV export -- alongside a configurable-column CSV parser and a cmd parser that shells out to an external script. Both ports fix two latent bugs in the originals: the sign character class rejected the typographic minus U+2212 that some PDF fonts emit, and NLB's hardcoded continuation indent broke when pdftotext compressed runs of spaces, so the threshold is now measured from the description column. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,179 @@
|
||||
// Package config loads the two hand-edited files the tool reads: the
|
||||
// data-root-wide rules.toml, and one account.toml per account folder.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
)
|
||||
|
||||
const (
|
||||
// RulesFile is the rules file at the root of the data directory.
|
||||
RulesFile = "rules.toml"
|
||||
// AccountFile is the per-account config inside each account folder.
|
||||
AccountFile = "account.toml"
|
||||
// StateDir holds the rebuildable SQLite index.
|
||||
StateDir = ".money"
|
||||
// IndexFile is the SQLite index inside StateDir.
|
||||
IndexFile = "index.db"
|
||||
)
|
||||
|
||||
// Rule is one entry in rules.toml. Rules are evaluated in file order and the
|
||||
// first one whose Match (and optional Account) matches wins.
|
||||
type Rule struct {
|
||||
Match string `toml:"match"`
|
||||
Tag string `toml:"tag"`
|
||||
Transfer bool `toml:"transfer"`
|
||||
Account string `toml:"account"` // optional: restrict to one account slug
|
||||
// Counterparty matches the other side's account number, which for
|
||||
// movements between the user's own accounts is often the only reliable
|
||||
// signal. Optional; when set, it must match as well as Match.
|
||||
Counterparty string `toml:"counterparty"`
|
||||
// Type matches the bank's own classification, e.g. Revolut's CARD_PAYMENT.
|
||||
// Optional; when set, it must match as well as Match.
|
||||
Type string `toml:"type"`
|
||||
}
|
||||
|
||||
// Rules is the parsed rules.toml.
|
||||
type Rules struct {
|
||||
Rule []Rule `toml:"rule"`
|
||||
}
|
||||
|
||||
// LoadRules reads rules.toml from the data root. A missing file is not an
|
||||
// error: it just means nothing is tagged automatically yet.
|
||||
func LoadRules(root string) (*Rules, error) {
|
||||
path := filepath.Join(root, RulesFile)
|
||||
var r Rules
|
||||
if _, err := toml.DecodeFile(path, &r); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &r, nil
|
||||
}
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
for i, rule := range r.Rule {
|
||||
if rule.Match == "" && rule.Counterparty == "" && rule.Type == "" {
|
||||
return nil, fmt.Errorf("%s: rule %d has no match, counterparty or type pattern", path, i+1)
|
||||
}
|
||||
if rule.Tag == "" && !rule.Transfer {
|
||||
return nil, fmt.Errorf("%s: rule %d (%q) sets neither tag nor transfer", path, i+1, rule.Match)
|
||||
}
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// Column locates one field in a CSV row.
|
||||
type Column struct {
|
||||
Col int `toml:"col"`
|
||||
Layout string `toml:"layout"` // date only, Go reference layout
|
||||
Decimal string `toml:"decimal"` // amount only, default "."
|
||||
Thousands string `toml:"thousands"` // amount only, default ""
|
||||
}
|
||||
|
||||
// CSVConfig describes how to read a delimited statement.
|
||||
type CSVConfig struct {
|
||||
Delimiter string `toml:"delimiter"`
|
||||
SkipRows int `toml:"skip_rows"`
|
||||
Encoding string `toml:"encoding"` // "" or "utf-8"; other encodings unsupported for now
|
||||
Date Column `toml:"date"`
|
||||
Description Column `toml:"description"`
|
||||
Amount *Column `toml:"amount"` // single signed column...
|
||||
Debit *Column `toml:"debit"` // ...or a debit/credit pair
|
||||
Credit *Column `toml:"credit"`
|
||||
// Invert flips the sign of the parsed amount, for statements that report
|
||||
// outflows as positive numbers.
|
||||
Invert bool `toml:"invert"`
|
||||
}
|
||||
|
||||
// CmdConfig runs an external extractor (e.g. one of the existing Python
|
||||
// scripts) and reads normalised CSV from its stdout.
|
||||
type CmdConfig struct {
|
||||
// Argv is the command to run. The literal token "{{file}}" is replaced
|
||||
// with the absolute path of the statement being imported.
|
||||
Argv []string `toml:"argv"`
|
||||
// Layout is the date layout the script emits; defaults to 2006-01-02.
|
||||
Layout string `toml:"layout"`
|
||||
// SkipRows skips leading rows of the script's output (e.g. a header).
|
||||
SkipRows int `toml:"skip_rows"`
|
||||
}
|
||||
|
||||
// Account is a parsed account.toml.
|
||||
type Account struct {
|
||||
Slug string // folder name, filled in by LoadAccounts
|
||||
Dir string // absolute path to the account folder
|
||||
Name string `toml:"name"`
|
||||
Currency string `toml:"currency"`
|
||||
MinorDigits *int `toml:"minor_digits"`
|
||||
Parser string `toml:"parser"`
|
||||
// Include restricts which files in the folder are treated as statements.
|
||||
// Defaults to every regular file except account.toml and dotfiles.
|
||||
Include []string `toml:"include"`
|
||||
CSV *CSVConfig `toml:"csv"`
|
||||
Cmd *CmdConfig `toml:"cmd"`
|
||||
}
|
||||
|
||||
// Digits returns the configured minor-unit scale, defaulting to 2.
|
||||
func (a *Account) Digits() int {
|
||||
if a.MinorDigits != nil {
|
||||
return *a.MinorDigits
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
// LoadAccounts finds every account folder under root. A folder is an account
|
||||
// if it contains an account.toml.
|
||||
func LoadAccounts(root string) ([]*Account, error) {
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read data root %s: %w", root, err)
|
||||
}
|
||||
var accounts []*Account
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() || strings.HasPrefix(e.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Join(root, e.Name())
|
||||
cfgPath := filepath.Join(dir, AccountFile)
|
||||
if _, err := os.Stat(cfgPath); err != nil {
|
||||
continue // not an account folder
|
||||
}
|
||||
a, err := loadAccount(dir, e.Name(), cfgPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accounts = append(accounts, a)
|
||||
}
|
||||
sort.Slice(accounts, func(i, j int) bool { return accounts[i].Slug < accounts[j].Slug })
|
||||
return accounts, nil
|
||||
}
|
||||
|
||||
func loadAccount(dir, slug, cfgPath string) (*Account, error) {
|
||||
var a Account
|
||||
if _, err := toml.DecodeFile(cfgPath, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", cfgPath, err)
|
||||
}
|
||||
a.Slug = slug
|
||||
a.Dir = dir
|
||||
if a.Name == "" {
|
||||
a.Name = slug
|
||||
}
|
||||
if a.Currency == "" {
|
||||
return nil, fmt.Errorf("%s: currency is required", cfgPath)
|
||||
}
|
||||
if a.Parser == "" {
|
||||
return nil, fmt.Errorf("%s: parser is required", cfgPath)
|
||||
}
|
||||
if a.Digits() < 0 || a.Digits() > 8 {
|
||||
return nil, fmt.Errorf("%s: minor_digits must be between 0 and 8", cfgPath)
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// IndexPath returns the location of the SQLite index for a data root.
|
||||
func IndexPath(root string) string {
|
||||
return filepath.Join(root, StateDir, IndexFile)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// Package glob implements the tiny wildcard language used by rules.toml.
|
||||
//
|
||||
// Only '*' (any run of characters, including none) and '?' (exactly one
|
||||
// character) are special; everything else matches literally. Unlike path.Match
|
||||
// there is no separator handling, because we match statement descriptions, not
|
||||
// paths. Matching is case-insensitive.
|
||||
package glob
|
||||
|
||||
import "strings"
|
||||
|
||||
// Match reports whether s matches pattern.
|
||||
func Match(pattern, s string) bool {
|
||||
return match([]rune(strings.ToUpper(pattern)), []rune(strings.ToUpper(s)))
|
||||
}
|
||||
|
||||
// match is the classic two-pointer wildcard algorithm: linear time, no
|
||||
// backtracking blowup on patterns like "*a*a*a*".
|
||||
func match(p, s []rune) bool {
|
||||
var (
|
||||
pi, si int
|
||||
star = -1 // index in p of the last '*' seen
|
||||
starMatch int // index in s that '*' was assumed to match up to
|
||||
hasStar bool
|
||||
)
|
||||
for si < len(s) {
|
||||
switch {
|
||||
case pi < len(p) && (p[pi] == '?' || p[pi] == s[si]):
|
||||
pi++
|
||||
si++
|
||||
case pi < len(p) && p[pi] == '*':
|
||||
star, hasStar = pi, true
|
||||
starMatch = si
|
||||
pi++
|
||||
case hasStar:
|
||||
// Backtrack: let the last '*' swallow one more character.
|
||||
starMatch++
|
||||
pi, si = star+1, starMatch
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
for pi < len(p) && p[pi] == '*' {
|
||||
pi++
|
||||
}
|
||||
return pi == len(p)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package glob
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMatch(t *testing.T) {
|
||||
cases := []struct {
|
||||
pattern, s string
|
||||
want bool
|
||||
}{
|
||||
{"*LIDL*", "CARD PAYMENT LIDL 1234 SOFIA", true},
|
||||
{"*LIDL*", "lidl", true},
|
||||
{"*lidl*", "CARD PAYMENT LIDL", true},
|
||||
{"LIDL", "LIDL", true},
|
||||
{"LIDL", "LIDL SOFIA", false},
|
||||
{"LIDL*", "LIDL SOFIA", true},
|
||||
{"*SOFIA", "LIDL SOFIA", true},
|
||||
{"*SOFIA", "LIDL SOFIA 1", false},
|
||||
{"", "", true},
|
||||
{"", "x", false},
|
||||
{"*", "", true},
|
||||
{"**", "anything", true},
|
||||
{"?", "a", true},
|
||||
{"?", "ab", false},
|
||||
{"a?c", "abc", true},
|
||||
{"a?c", "ac", false},
|
||||
{"*A*B*C*", "xxAyyBzzCww", true},
|
||||
{"*A*B*C*", "xxAyyCzzBww", false},
|
||||
{"*TO SAVINGS*", "TRANSFER TO SAVINGS ACC", true},
|
||||
{"*a*a*a*", "aaaaaaaaaaaaaaaaaaaaaaaaaaaab", true},
|
||||
// Pathological for naive recursion: must return quickly, not hang.
|
||||
{"a*a*a*a*a*a*a*a*b", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := Match(c.pattern, c.s); got != c.want {
|
||||
t.Errorf("Match(%q, %q) = %v, want %v", c.pattern, c.s, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchUnicode(t *testing.T) {
|
||||
if !Match("*КАУФЛАНД*", "плащане кауфланд софия") {
|
||||
t.Error("expected case-insensitive unicode match")
|
||||
}
|
||||
if !Match("ка?фланд", "КАУФЛАНД") {
|
||||
t.Error("expected '?' to match one rune, not one byte")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
// Package importer walks the data root, extracts transactions from every
|
||||
// statement file, and inserts the ones not already in the index.
|
||||
package importer
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
"git.petrovv.com/nikola/money/internal/glob"
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
"git.petrovv.com/nikola/money/internal/parser"
|
||||
"git.petrovv.com/nikola/money/internal/rules"
|
||||
"git.petrovv.com/nikola/money/internal/store"
|
||||
)
|
||||
|
||||
// FileResult is what happened to one statement file.
|
||||
type FileResult struct {
|
||||
Account string
|
||||
Path string // relative to the data root
|
||||
Parsed int
|
||||
New int
|
||||
Skipped int // already present, i.e. deduplicated
|
||||
Err error
|
||||
// Warnings are non-fatal notes: rows the parser deliberately dropped, and
|
||||
// breaks in the statement's balance chain.
|
||||
Warnings []string
|
||||
}
|
||||
|
||||
// Result summarises a whole import run.
|
||||
type Result struct {
|
||||
Files []FileResult
|
||||
Retagged int
|
||||
}
|
||||
|
||||
// Total counts new rows across every file.
|
||||
func (r Result) Total() (parsed, added, skipped int) {
|
||||
for _, f := range r.Files {
|
||||
parsed += f.Parsed
|
||||
added += f.New
|
||||
skipped += f.Skipped
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Errs collects per-file failures. One unreadable statement must not abort the
|
||||
// whole run, so errors are reported rather than returned.
|
||||
func (r Result) Errs() []FileResult {
|
||||
var out []FileResult
|
||||
for _, f := range r.Files {
|
||||
if f.Err != nil {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Options tunes an import run.
|
||||
type Options struct {
|
||||
// Force reimports statement files whose contents are unchanged. Without
|
||||
// it, files with a matching checksum are skipped entirely.
|
||||
Force bool
|
||||
}
|
||||
|
||||
// Run imports every account under root into db and then applies the rules.
|
||||
func Run(root string, db *store.DB, accounts []*config.Account, engine *rules.Engine, opts Options) (Result, error) {
|
||||
var res Result
|
||||
for _, acc := range accounts {
|
||||
accountID, err := db.UpsertAccount(model.Account{
|
||||
Slug: acc.Slug,
|
||||
Name: acc.Name,
|
||||
Currency: acc.Currency,
|
||||
MinorDigits: acc.Digits(),
|
||||
})
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
p, err := parser.For(acc)
|
||||
if err != nil {
|
||||
// A misconfigured account is worth reporting, but the other
|
||||
// accounts should still import.
|
||||
res.Files = append(res.Files, FileResult{Account: acc.Slug, Path: acc.Slug, Err: err})
|
||||
continue
|
||||
}
|
||||
|
||||
files, err := statementFiles(acc)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
for _, path := range files {
|
||||
fr := importFile(root, db, acc, accountID, p, engine, path, opts)
|
||||
res.Files = append(res.Files, fr)
|
||||
}
|
||||
}
|
||||
|
||||
n, err := engine.Retag(db)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Retagged = n
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func importFile(root string, db *store.DB, acc *config.Account, accountID int64,
|
||||
p parser.Parser, engine *rules.Engine, path string, opts Options) FileResult {
|
||||
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
rel = path
|
||||
}
|
||||
fr := FileResult{Account: acc.Slug, Path: rel}
|
||||
|
||||
sum, err := checksum(path)
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
if !opts.Force {
|
||||
prev, seen, err := db.SourceFileSHA(accountID, rel)
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
if seen && prev == sum {
|
||||
return fr // unchanged since last import; nothing to do
|
||||
}
|
||||
}
|
||||
|
||||
txns, err := p.Parse(path, acc)
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
fr.Parsed = len(txns)
|
||||
|
||||
if w, ok := p.(parser.Warner); ok {
|
||||
fr.Warnings = append(fr.Warnings, w.Warnings()...)
|
||||
}
|
||||
fr.Warnings = append(fr.Warnings, checkBalances(txns, acc.Digits())...)
|
||||
|
||||
sourceID, err := db.SourceFile(accountID, rel, sum, time.Now().UTC().Format(time.RFC3339))
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
|
||||
// Identical lines within one statement (two coffees on the same day) are
|
||||
// distinguished by their ordinal, so both survive; the same line seen
|
||||
// again in an overlapping statement gets the same fingerprint and is
|
||||
// deduplicated.
|
||||
seen := map[string]int{}
|
||||
for _, t := range txns {
|
||||
key := fingerprintKey(t)
|
||||
ordinal := seen[key]
|
||||
seen[key]++
|
||||
|
||||
txn := model.Transaction{
|
||||
AccountID: accountID,
|
||||
SourceFileID: sourceID,
|
||||
Fingerprint: fingerprint(key, ordinal),
|
||||
Date: t.Date,
|
||||
Description: t.Description,
|
||||
AmountMinor: t.AmountMinor,
|
||||
Counterparty: t.Counterparty,
|
||||
Type: t.Type,
|
||||
BalanceMinor: t.BalanceMinor,
|
||||
}
|
||||
txn.RuleTag, txn.RuleTransfer = engine.ApplyTxn(acc.Slug, txn)
|
||||
added, err := db.InsertTransaction(txn)
|
||||
if err != nil {
|
||||
fr.Err = err
|
||||
return fr
|
||||
}
|
||||
if added {
|
||||
fr.New++
|
||||
} else {
|
||||
fr.Skipped++
|
||||
}
|
||||
}
|
||||
return fr
|
||||
}
|
||||
|
||||
// checkBalances verifies that each reported balance is the previous one plus
|
||||
// the transaction amount, which is what the original extraction scripts did.
|
||||
// A break means a row was missed or misparsed, so it is worth saying out loud
|
||||
// even though the import still proceeds.
|
||||
func checkBalances(txns []parser.RawTxn, digits int) []string {
|
||||
var warnings []string
|
||||
var prev *parser.RawTxn
|
||||
|
||||
for i := range txns {
|
||||
t := &txns[i]
|
||||
if t.BalanceMinor == nil {
|
||||
continue // this statement does not report running balances
|
||||
}
|
||||
if prev != nil {
|
||||
expected := *prev.BalanceMinor + t.AmountMinor
|
||||
if expected != *t.BalanceMinor {
|
||||
warnings = append(warnings, fmt.Sprintf(
|
||||
"balance chain breaks at %s %q: statement says %s, previous balance plus amount is %s",
|
||||
t.Date, truncate(t.Description, 40),
|
||||
model.FormatMinor(*t.BalanceMinor, digits),
|
||||
model.FormatMinor(expected, digits)))
|
||||
}
|
||||
}
|
||||
prev = t
|
||||
}
|
||||
return warnings
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n]) + "…"
|
||||
}
|
||||
|
||||
func fingerprintKey(t parser.RawTxn) string {
|
||||
return strings.Join([]string{
|
||||
t.Date,
|
||||
fmt.Sprintf("%d", t.AmountMinor),
|
||||
model.NormalizeDescription(t.Description),
|
||||
}, "\x00")
|
||||
}
|
||||
|
||||
func fingerprint(key string, ordinal int) string {
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("%s\x00%d", key, ordinal)))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// statementFiles lists the files in an account folder that should be parsed:
|
||||
// every regular file except account.toml and dotfiles, narrowed by the
|
||||
// account's optional include globs.
|
||||
func statementFiles(acc *config.Account) ([]string, error) {
|
||||
entries, err := os.ReadDir(acc.Dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read account dir %s: %w", acc.Dir, err)
|
||||
}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if e.IsDir() || strings.HasPrefix(name, ".") || name == config.AccountFile {
|
||||
continue
|
||||
}
|
||||
if len(acc.Include) > 0 && !matchAny(acc.Include, name) {
|
||||
continue
|
||||
}
|
||||
out = append(out, filepath.Join(acc.Dir, name))
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func matchAny(patterns []string, name string) bool {
|
||||
for _, p := range patterns {
|
||||
if glob.Match(p, name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func checksum(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
"git.petrovv.com/nikola/money/internal/parser"
|
||||
"git.petrovv.com/nikola/money/internal/rules"
|
||||
"git.petrovv.com/nikola/money/internal/store"
|
||||
)
|
||||
|
||||
const accountTOML = `
|
||||
name = "Checking"
|
||||
currency = "EUR"
|
||||
parser = "csv"
|
||||
[csv]
|
||||
skip_rows = 1
|
||||
date = { col = 0, layout = "2006-01-02" }
|
||||
description = { col = 1 }
|
||||
amount = { col = 2 }
|
||||
`
|
||||
|
||||
// newRoot builds a data root with one account and the given statement files.
|
||||
func newRoot(t *testing.T, statements map[string]string) (string, *store.DB, []*config.Account, *rules.Engine) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
dir := filepath.Join(root, "checking")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write(t, filepath.Join(dir, config.AccountFile), accountTOML)
|
||||
for name, body := range statements {
|
||||
write(t, filepath.Join(dir, name), body)
|
||||
}
|
||||
|
||||
db, err := store.Open(config.IndexPath(root))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
|
||||
accounts, err := config.LoadAccounts(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := rules.New(&config.Rules{Rule: []config.Rule{{Match: "*LIDL*", Tag: "groceries"}}})
|
||||
return root, db, accounts, engine
|
||||
}
|
||||
|
||||
func write(t *testing.T, path, body string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRun(t *testing.T, root string, db *store.DB, accounts []*config.Account, e *rules.Engine, opts Options) Result {
|
||||
t.Helper()
|
||||
res, err := Run(root, db, accounts, e, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range res.Errs() {
|
||||
t.Fatalf("import of %s failed: %v", f.Path, f.Err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Identical lines inside one statement are distinct transactions; the same
|
||||
// line seen again in an overlapping statement is not.
|
||||
func TestDedupe(t *testing.T) {
|
||||
root, db, accounts, engine := newRoot(t, map[string]string{
|
||||
"2026-01.csv": `date,description,amount
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
2026-01-10,RENT,-800.00
|
||||
`,
|
||||
})
|
||||
|
||||
res := mustRun(t, root, db, accounts, engine, Options{})
|
||||
if _, added, _ := res.Total(); added != 3 {
|
||||
t.Fatalf("first import added %d rows, want 3 (identical same-day lines must both survive)", added)
|
||||
}
|
||||
|
||||
// Unchanged file: skipped without even parsing.
|
||||
res = mustRun(t, root, db, accounts, engine, Options{})
|
||||
if parsed, added, _ := res.Total(); parsed != 0 || added != 0 {
|
||||
t.Errorf("re-import parsed %d and added %d, want 0 and 0", parsed, added)
|
||||
}
|
||||
|
||||
// Same file, parsed again: every row is recognised as a duplicate.
|
||||
res = mustRun(t, root, db, accounts, engine, Options{Force: true})
|
||||
if _, added, skipped := res.Total(); added != 0 || skipped != 3 {
|
||||
t.Errorf("forced re-import added %d, skipped %d; want 0 and 3", added, skipped)
|
||||
}
|
||||
|
||||
// An overlapping statement contributes only its genuinely new rows.
|
||||
write(t, filepath.Join(root, "checking", "2026-02.csv"), `date,description,amount
|
||||
2026-01-10,RENT,-800.00
|
||||
2026-02-10,RENT,-800.00
|
||||
`)
|
||||
accounts, err := config.LoadAccounts(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res = mustRun(t, root, db, accounts, engine, Options{})
|
||||
if _, added, skipped := res.Total(); added != 1 || skipped != 1 {
|
||||
t.Errorf("overlapping import added %d, skipped %d; want 1 and 1", added, skipped)
|
||||
}
|
||||
|
||||
txns, err := db.Transactions(store.Filter{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 4 {
|
||||
t.Errorf("index holds %d transactions, want 4", len(txns))
|
||||
}
|
||||
}
|
||||
|
||||
// Rules are applied as rows are inserted, so a fresh import is already tagged.
|
||||
func TestImportAppliesRules(t *testing.T) {
|
||||
root, db, accounts, engine := newRoot(t, map[string]string{
|
||||
"2026-01.csv": `date,description,amount
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
2026-01-10,RENT,-800.00
|
||||
`,
|
||||
})
|
||||
res := mustRun(t, root, db, accounts, engine, Options{})
|
||||
if res.Retagged != 0 {
|
||||
t.Errorf("retag changed %d rows after import, want 0: rules should already be applied", res.Retagged)
|
||||
}
|
||||
|
||||
untagged, err := db.Transactions(store.Filter{Untagged: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(untagged) != 1 || untagged[0].Description != "RENT" {
|
||||
t.Errorf("untagged = %+v, want only RENT", untagged)
|
||||
}
|
||||
}
|
||||
|
||||
// A broken statement must be reported without aborting the rest of the run.
|
||||
func TestBadFileIsReportedNotFatal(t *testing.T) {
|
||||
root, db, accounts, engine := newRoot(t, map[string]string{
|
||||
"good.csv": `date,description,amount
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
`,
|
||||
"bad.csv": `date,description,amount
|
||||
not-a-date,BROKEN,-1.00
|
||||
`,
|
||||
})
|
||||
|
||||
res, err := Run(root, db, accounts, engine, Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Run returned a fatal error, want a per-file report: %v", err)
|
||||
}
|
||||
failures := res.Errs()
|
||||
if len(failures) != 1 || filepath.Base(failures[0].Path) != "bad.csv" {
|
||||
t.Fatalf("failures = %+v, want exactly bad.csv", failures)
|
||||
}
|
||||
if _, added, _ := res.Total(); added != 1 {
|
||||
t.Errorf("added %d rows, want 1 from good.csv", added)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckBalances(t *testing.T) {
|
||||
balance := func(v int64) *int64 { return &v }
|
||||
|
||||
good := []parser.RawTxn{
|
||||
{Date: "2026-01-01", Description: "A", AmountMinor: -1000, BalanceMinor: balance(9000)},
|
||||
{Date: "2026-01-02", Description: "B", AmountMinor: -500, BalanceMinor: balance(8500)},
|
||||
{Date: "2026-01-03", Description: "C", AmountMinor: 2000, BalanceMinor: balance(10500)},
|
||||
}
|
||||
if w := checkBalances(good, 2); len(w) != 0 {
|
||||
t.Errorf("a consistent chain produced warnings: %v", w)
|
||||
}
|
||||
|
||||
// A missed row shows up as a break at the row after it.
|
||||
broken := []parser.RawTxn{
|
||||
{Date: "2026-01-01", Description: "A", AmountMinor: -1000, BalanceMinor: balance(9000)},
|
||||
{Date: "2026-01-02", Description: "B", AmountMinor: -500, BalanceMinor: balance(7000)},
|
||||
}
|
||||
w := checkBalances(broken, 2)
|
||||
if len(w) != 1 {
|
||||
t.Fatalf("got %d warnings, want 1: %v", len(w), w)
|
||||
}
|
||||
if !strings.Contains(w[0], "70.00") || !strings.Contains(w[0], "85.00") {
|
||||
t.Errorf("warning = %q, want both the reported and the expected balance", w[0])
|
||||
}
|
||||
|
||||
// Statements that report no balances are not checked.
|
||||
none := []parser.RawTxn{
|
||||
{Date: "2026-01-01", Description: "A", AmountMinor: -1000},
|
||||
{Date: "2026-01-02", Description: "B", AmountMinor: -500},
|
||||
}
|
||||
if w := checkBalances(none, 2); len(w) != 0 {
|
||||
t.Errorf("statements without balances produced warnings: %v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// Only files matching include globs are treated as statements.
|
||||
func TestIncludeGlobs(t *testing.T) {
|
||||
root, db, accounts, engine := newRoot(t, map[string]string{
|
||||
"2026-01.csv": `date,description,amount
|
||||
2026-01-06,LIDL SOFIA,-45.20
|
||||
`,
|
||||
"notes.txt": "not a statement",
|
||||
})
|
||||
accounts[0].Include = []string{"*.csv"}
|
||||
|
||||
res := mustRun(t, root, db, accounts, engine, Options{})
|
||||
if len(res.Files) != 1 {
|
||||
t.Fatalf("processed %d files, want 1: %+v", len(res.Files), res.Files)
|
||||
}
|
||||
if _, added, _ := res.Total(); added != 1 {
|
||||
t.Errorf("added %d rows, want 1", added)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
// Package model holds the core domain types shared by every other package.
|
||||
package model
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Account is one folder in the data root.
|
||||
type Account struct {
|
||||
ID int64
|
||||
Slug string // folder name, unique
|
||||
Name string // human readable, from account.toml
|
||||
Currency string
|
||||
MinorDigits int
|
||||
}
|
||||
|
||||
// Transaction is a single line extracted from a statement, plus whatever the
|
||||
// rules engine and the user have said about it.
|
||||
//
|
||||
// Amount is stored in minor units (cents) and signed: negative is an outflow.
|
||||
type Transaction struct {
|
||||
ID int64
|
||||
AccountID int64
|
||||
AccountSlug string
|
||||
Currency string
|
||||
MinorDigits int
|
||||
|
||||
Fingerprint string
|
||||
Date string // YYYY-MM-DD
|
||||
Description string
|
||||
AmountMinor int64
|
||||
SourceFileID int64
|
||||
SourcePath string
|
||||
|
||||
// Counterparty is the other side's account number (an IBAN, where the
|
||||
// statement gives one). Transfers between the user's own accounts are
|
||||
// often only distinguishable by it.
|
||||
Counterparty string
|
||||
// Type is the bank's own classification, e.g. Revolut's CARD_PAYMENT.
|
||||
Type string
|
||||
// BalanceMinor is the running balance the statement reported after this
|
||||
// transaction, when it reports one. Used to verify the balance chain.
|
||||
BalanceMinor *int64
|
||||
|
||||
RuleTag string
|
||||
ManualTag string
|
||||
RuleTransfer bool
|
||||
ManualTransfer *bool
|
||||
}
|
||||
|
||||
// Tag is the effective category: a manual tag always beats a rule tag.
|
||||
func (t Transaction) Tag() string {
|
||||
if t.ManualTag != "" {
|
||||
return t.ManualTag
|
||||
}
|
||||
return t.RuleTag
|
||||
}
|
||||
|
||||
// IsTransfer reports whether this line moves money between the user's own
|
||||
// accounts and so must not count as spending. A manual override wins.
|
||||
func (t Transaction) IsTransfer() bool {
|
||||
if t.ManualTransfer != nil {
|
||||
return *t.ManualTransfer
|
||||
}
|
||||
return t.RuleTransfer
|
||||
}
|
||||
|
||||
// FormatAmount renders the amount using the account's minor-unit scale.
|
||||
func (t Transaction) FormatAmount() string {
|
||||
return FormatMinor(t.AmountMinor, t.MinorDigits)
|
||||
}
|
||||
|
||||
// FormatMinor renders minor units as a decimal string, e.g. -4520/2 -> "-45.20".
|
||||
func FormatMinor(minor int64, digits int) string {
|
||||
if digits <= 0 {
|
||||
return fmt.Sprintf("%d", minor)
|
||||
}
|
||||
neg := minor < 0
|
||||
if neg {
|
||||
minor = -minor
|
||||
}
|
||||
scale := int64(1)
|
||||
for i := 0; i < digits; i++ {
|
||||
scale *= 10
|
||||
}
|
||||
whole, frac := minor/scale, minor%scale
|
||||
s := fmt.Sprintf("%d.%0*d", whole, digits, frac)
|
||||
if neg {
|
||||
s = "-" + s
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// NormalizeDescription collapses whitespace and upper-cases, so that
|
||||
// fingerprints and glob matching are insensitive to statement formatting noise.
|
||||
func NormalizeDescription(s string) string {
|
||||
return strings.ToUpper(strings.Join(strings.Fields(s), " "))
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ParseAmount converts a decimal string from a statement into signed minor
|
||||
// units. It is deliberately forgiving, because statements are messy:
|
||||
//
|
||||
// "1.234,56" (decimal ",", thousands ".") -> 123456
|
||||
// "-45.20" -> -4520
|
||||
// "45,20-" (trailing minus) -> -4520
|
||||
// "(45.20)" (accounting negative) -> -4520
|
||||
// "1 234.56 EUR" -> 123456
|
||||
//
|
||||
// decimal defaults to "."; every other separator is discarded, so thousands is
|
||||
// only a readability hint in account.toml and needs no special handling here.
|
||||
// digits is the account's minor-unit scale; extra fraction digits are rounded
|
||||
// half away from zero.
|
||||
func ParseAmount(s, decimal, thousands string, digits int) (int64, error) {
|
||||
_ = thousands // dropped along with all other non-decimal separators
|
||||
orig := s
|
||||
if decimal == "" {
|
||||
decimal = "."
|
||||
}
|
||||
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0, fmt.Errorf("empty amount")
|
||||
}
|
||||
|
||||
neg := false
|
||||
if strings.HasPrefix(s, "(") && strings.HasSuffix(s, ")") {
|
||||
neg = true
|
||||
s = strings.TrimSuffix(strings.TrimPrefix(s, "("), ")")
|
||||
}
|
||||
if strings.HasSuffix(s, "-") {
|
||||
neg = true
|
||||
s = strings.TrimSuffix(s, "-")
|
||||
}
|
||||
|
||||
// Keep only digits and the decimal separator. Everything else -- currency
|
||||
// codes, symbols, thin and non-breaking spaces, thousands separators --
|
||||
// is noise. A minus anywhere flips the sign; a plus is ignored.
|
||||
var b strings.Builder
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-' || r == '−':
|
||||
neg = !neg
|
||||
case string(r) == decimal:
|
||||
b.WriteRune('.')
|
||||
}
|
||||
}
|
||||
clean := b.String()
|
||||
if clean == "" || clean == "." {
|
||||
return 0, fmt.Errorf("cannot parse amount %q", orig)
|
||||
}
|
||||
|
||||
intPart, fracPart, _ := strings.Cut(clean, ".")
|
||||
if strings.Contains(fracPart, ".") {
|
||||
return 0, fmt.Errorf("cannot parse amount %q: multiple decimal separators", orig)
|
||||
}
|
||||
if intPart == "" {
|
||||
intPart = "0"
|
||||
}
|
||||
|
||||
whole, err := strconv.ParseInt(intPart, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse amount %q: %w", orig, err)
|
||||
}
|
||||
|
||||
scale := int64(1)
|
||||
for range digits {
|
||||
scale *= 10
|
||||
}
|
||||
|
||||
var frac int64
|
||||
if digits > 0 {
|
||||
switch {
|
||||
case len(fracPart) > digits:
|
||||
// Round half away from zero on the first dropped digit.
|
||||
kept, err := strconv.ParseInt(fracPart[:digits], 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse amount %q: %w", orig, err)
|
||||
}
|
||||
frac = kept
|
||||
if fracPart[digits] >= '5' {
|
||||
frac++
|
||||
}
|
||||
case fracPart != "":
|
||||
padded := fracPart + strings.Repeat("0", digits-len(fracPart))
|
||||
if frac, err = strconv.ParseInt(padded, 10, 64); err != nil {
|
||||
return 0, fmt.Errorf("cannot parse amount %q: %w", orig, err)
|
||||
}
|
||||
}
|
||||
} else if fracPart != "" && fracPart[0] >= '5' {
|
||||
whole++
|
||||
}
|
||||
|
||||
v := whole*scale + frac
|
||||
if neg {
|
||||
v = -v
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package parser
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseAmount(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
decimal string
|
||||
thousands string
|
||||
digits int
|
||||
want int64
|
||||
}{
|
||||
{"45.20", ".", "", 2, 4520},
|
||||
{"-45.20", ".", "", 2, -4520},
|
||||
{"45,20", ",", ".", 2, 4520},
|
||||
{"1.234,56", ",", ".", 2, 123456},
|
||||
{"1,234.56", ".", ",", 2, 123456},
|
||||
{"1 234.56", ".", " ", 2, 123456},
|
||||
{"45.20 EUR", ".", "", 2, 4520},
|
||||
{"BGN 45.20", ".", "", 2, 4520},
|
||||
{"45,20-", ",", "", 2, -4520},
|
||||
{"(45.20)", ".", "", 2, -4520},
|
||||
{"+45.20", ".", "", 2, 4520},
|
||||
{"0.00", ".", "", 2, 0},
|
||||
{".50", ".", "", 2, 50},
|
||||
{"45", ".", "", 2, 4500},
|
||||
{"45.2", ".", "", 2, 4520},
|
||||
{"45.205", ".", "", 2, 4521}, // round half away from zero
|
||||
{"45.204", ".", "", 2, 4520}, // round down
|
||||
{"-45.205", ".", "", 2, -4521},
|
||||
{"1234", ".", "", 0, 1234}, // zero-digit currency
|
||||
{"1234.6", ".", "", 0, 1235},
|
||||
{"1.234567", ".", "", 3, 1235},
|
||||
{"−45.20", ".", "", 2, -4520}, // U+2212 minus
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := ParseAmount(c.in, c.decimal, c.thousands, c.digits)
|
||||
if err != nil {
|
||||
t.Errorf("ParseAmount(%q) error: %v", c.in, err)
|
||||
continue
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("ParseAmount(%q, decimal=%q, digits=%d) = %d, want %d",
|
||||
c.in, c.decimal, c.digits, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAmountErrors(t *testing.T) {
|
||||
for _, in := range []string{"", " ", "abc", "-", "."} {
|
||||
if v, err := ParseAmount(in, ".", "", 2); err == nil {
|
||||
t.Errorf("ParseAmount(%q) = %d, want error", in, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register("cmd", newCmdParser)
|
||||
}
|
||||
|
||||
// FileToken is replaced with the statement's absolute path in a cmd parser's argv.
|
||||
const FileToken = "{{file}}"
|
||||
|
||||
// cmdRunTimeout bounds an extractor run so a hung script cannot wedge an import.
|
||||
const cmdRunTimeout = 2 * time.Minute
|
||||
|
||||
// cmdParser runs an external extractor and reads normalised CSV from its
|
||||
// stdout: date,description,amount with any further columns ignored. This is
|
||||
// the bridge that lets the existing Python extractors be used unchanged.
|
||||
type cmdParser struct {
|
||||
cfg config.CmdConfig
|
||||
digits int
|
||||
}
|
||||
|
||||
func newCmdParser(acc *config.Account) (Parser, error) {
|
||||
if acc.Cmd == nil || len(acc.Cmd.Argv) == 0 {
|
||||
return nil, fmt.Errorf("account %s: parser \"cmd\" requires [cmd] with a non-empty argv", acc.Slug)
|
||||
}
|
||||
cfg := *acc.Cmd
|
||||
if cfg.Layout == "" {
|
||||
cfg.Layout = "2006-01-02"
|
||||
}
|
||||
if !hasFileToken(cfg.Argv) {
|
||||
return nil, fmt.Errorf("account %s: [cmd] argv must contain %s so the script knows which file to read",
|
||||
acc.Slug, FileToken)
|
||||
}
|
||||
return &cmdParser{cfg: cfg, digits: acc.Digits()}, nil
|
||||
}
|
||||
|
||||
func hasFileToken(argv []string) bool {
|
||||
for _, a := range argv {
|
||||
if strings.Contains(a, FileToken) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *cmdParser) Parse(path string, acc *config.Account) ([]RawTxn, error) {
|
||||
argv := make([]string, len(p.cfg.Argv))
|
||||
for i, a := range p.cfg.Argv {
|
||||
argv[i] = strings.ReplaceAll(a, FileToken, path)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cmdRunTimeout)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
|
||||
cmd.Dir = acc.Dir
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
msg := strings.TrimSpace(stderr.String())
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return nil, fmt.Errorf("extractor %v timed out after %s", argv, cmdRunTimeout)
|
||||
}
|
||||
if msg != "" {
|
||||
return nil, fmt.Errorf("extractor %v failed: %w: %s", argv, err, msg)
|
||||
}
|
||||
return nil, fmt.Errorf("extractor %v failed: %w", argv, err)
|
||||
}
|
||||
|
||||
return p.parseOutput(&stdout, argv)
|
||||
}
|
||||
|
||||
func (p *cmdParser) parseOutput(out io.Reader, argv []string) ([]RawTxn, error) {
|
||||
r := csv.NewReader(out)
|
||||
r.FieldsPerRecord = -1
|
||||
r.LazyQuotes = true
|
||||
|
||||
var txns []RawTxn
|
||||
for row := 0; ; row++ {
|
||||
rec, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extractor %v: output row %d: %w", argv, row+1, err)
|
||||
}
|
||||
if row < p.cfg.SkipRows || isBlank(rec) {
|
||||
continue
|
||||
}
|
||||
if len(rec) < 3 {
|
||||
return nil, fmt.Errorf("extractor %v: output row %d has %d columns, want date,description,amount",
|
||||
argv, row+1, len(rec))
|
||||
}
|
||||
d, err := time.Parse(p.cfg.Layout, strings.TrimSpace(rec[0]))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extractor %v: output row %d: date %q does not match layout %q",
|
||||
argv, row+1, rec[0], p.cfg.Layout)
|
||||
}
|
||||
amount, err := ParseAmount(rec[2], ".", "", p.digits)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("extractor %v: output row %d: %w", argv, row+1, err)
|
||||
}
|
||||
txns = append(txns, RawTxn{
|
||||
Date: d.Format("2006-01-02"),
|
||||
Description: strings.TrimSpace(rec[1]),
|
||||
AmountMinor: amount,
|
||||
})
|
||||
}
|
||||
return txns, nil
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register("csv", newCSVParser)
|
||||
}
|
||||
|
||||
// csvParser reads a delimited statement using column positions from
|
||||
// account.toml. Amounts come either from one signed column, or from a
|
||||
// debit/credit pair.
|
||||
type csvParser struct {
|
||||
cfg config.CSVConfig
|
||||
digits int
|
||||
}
|
||||
|
||||
func newCSVParser(acc *config.Account) (Parser, error) {
|
||||
if acc.CSV == nil {
|
||||
return nil, fmt.Errorf("account %s: parser \"csv\" requires a [csv] section", acc.Slug)
|
||||
}
|
||||
cfg := *acc.CSV
|
||||
if cfg.Amount == nil && cfg.Debit == nil && cfg.Credit == nil {
|
||||
return nil, fmt.Errorf("account %s: [csv] needs either amount or debit/credit columns", acc.Slug)
|
||||
}
|
||||
if cfg.Amount != nil && (cfg.Debit != nil || cfg.Credit != nil) {
|
||||
return nil, fmt.Errorf("account %s: [csv] sets both amount and debit/credit; pick one", acc.Slug)
|
||||
}
|
||||
if cfg.Date.Layout == "" {
|
||||
return nil, fmt.Errorf("account %s: [csv] date needs a layout, e.g. layout = \"02.01.2006\"", acc.Slug)
|
||||
}
|
||||
return &csvParser{cfg: cfg, digits: acc.Digits()}, nil
|
||||
}
|
||||
|
||||
func (p *csvParser) Parse(path string, acc *config.Account) ([]RawTxn, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
r := csv.NewReader(f)
|
||||
r.FieldsPerRecord = -1 // statements are ragged more often than not
|
||||
r.LazyQuotes = true
|
||||
if d := p.cfg.Delimiter; d != "" {
|
||||
runes := []rune(d)
|
||||
if len(runes) != 1 {
|
||||
return nil, fmt.Errorf("delimiter %q must be a single character", d)
|
||||
}
|
||||
r.Comma = runes[0]
|
||||
}
|
||||
|
||||
var out []RawTxn
|
||||
for row := 0; ; row++ {
|
||||
rec, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: row %d: %w", path, row+1, err)
|
||||
}
|
||||
if row < p.cfg.SkipRows {
|
||||
continue
|
||||
}
|
||||
if isBlank(rec) {
|
||||
continue
|
||||
}
|
||||
txn, err := p.row(rec)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: row %d: %w", path, row+1, err)
|
||||
}
|
||||
out = append(out, txn)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *csvParser) row(rec []string) (RawTxn, error) {
|
||||
var t RawTxn
|
||||
|
||||
raw, err := field(rec, p.cfg.Date.Col)
|
||||
if err != nil {
|
||||
return t, fmt.Errorf("date column: %w", err)
|
||||
}
|
||||
d, err := time.Parse(p.cfg.Date.Layout, strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return t, fmt.Errorf("date %q does not match layout %q", raw, p.cfg.Date.Layout)
|
||||
}
|
||||
t.Date = d.Format("2006-01-02")
|
||||
|
||||
desc, err := field(rec, p.cfg.Description.Col)
|
||||
if err != nil {
|
||||
return t, fmt.Errorf("description column: %w", err)
|
||||
}
|
||||
t.Description = strings.TrimSpace(desc)
|
||||
|
||||
switch {
|
||||
case p.cfg.Amount != nil:
|
||||
raw, err := field(rec, p.cfg.Amount.Col)
|
||||
if err != nil {
|
||||
return t, fmt.Errorf("amount column: %w", err)
|
||||
}
|
||||
v, err := ParseAmount(raw, p.cfg.Amount.Decimal, p.cfg.Amount.Thousands, p.digits)
|
||||
if err != nil {
|
||||
return t, err
|
||||
}
|
||||
t.AmountMinor = v
|
||||
default:
|
||||
// Debit/credit pair: exactly one of the two carries a value, and both
|
||||
// are written as positive numbers.
|
||||
debit, err := p.optional(rec, p.cfg.Debit)
|
||||
if err != nil {
|
||||
return t, fmt.Errorf("debit column: %w", err)
|
||||
}
|
||||
credit, err := p.optional(rec, p.cfg.Credit)
|
||||
if err != nil {
|
||||
return t, fmt.Errorf("credit column: %w", err)
|
||||
}
|
||||
if debit != 0 && credit != 0 {
|
||||
return t, fmt.Errorf("both debit (%d) and credit (%d) are set", debit, credit)
|
||||
}
|
||||
t.AmountMinor = credit - abs(debit)
|
||||
}
|
||||
|
||||
if p.cfg.Invert {
|
||||
t.AmountMinor = -t.AmountMinor
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// optional parses a column that may legitimately be blank, as debit/credit
|
||||
// columns always are for half the rows.
|
||||
func (p *csvParser) optional(rec []string, c *config.Column) (int64, error) {
|
||||
if c == nil {
|
||||
return 0, nil
|
||||
}
|
||||
raw, err := field(rec, c.Col)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return 0, nil
|
||||
}
|
||||
return ParseAmount(raw, c.Decimal, c.Thousands, p.digits)
|
||||
}
|
||||
|
||||
func field(rec []string, i int) (string, error) {
|
||||
if i < 0 || i >= len(rec) {
|
||||
return "", fmt.Errorf("index %d out of range, row has %d columns", i, len(rec))
|
||||
}
|
||||
return rec[i], nil
|
||||
}
|
||||
|
||||
func isBlank(rec []string) bool {
|
||||
for _, f := range rec {
|
||||
if strings.TrimSpace(f) != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func abs(v int64) int64 {
|
||||
if v < 0 {
|
||||
return -v
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
)
|
||||
|
||||
func writeFile(t *testing.T, name, content string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), name)
|
||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestCSVSignedAmountColumn(t *testing.T) {
|
||||
path := writeFile(t, "st.csv", `Date,Ref,Description,Amount
|
||||
02.01.2026,X1,LIDL SOFIA 1234,"-45,20"
|
||||
03.01.2026,X2,ACME PAYROLL,"1.500,00"
|
||||
|
||||
04.01.2026,X3,COFFEE,"-3,50"
|
||||
`)
|
||||
acc := &config.Account{Slug: "checking", Currency: "EUR", Parser: "csv", CSV: &config.CSVConfig{
|
||||
SkipRows: 1,
|
||||
Date: config.Column{Col: 0, Layout: "02.01.2006"},
|
||||
Description: config.Column{Col: 2},
|
||||
Amount: &config.Column{Col: 3, Decimal: ",", Thousands: "."},
|
||||
}}
|
||||
p, err := For(acc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := p.Parse(path, acc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []RawTxn{
|
||||
{Date: "2026-01-02", Description: "LIDL SOFIA 1234", AmountMinor: -4520},
|
||||
{Date: "2026-01-03", Description: "ACME PAYROLL", AmountMinor: 150000},
|
||||
{Date: "2026-01-04", Description: "COFFEE", AmountMinor: -350},
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("got %d txns, want %d: %+v", len(got), len(want), got)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Errorf("txn %d = %+v, want %+v", i, got[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVDebitCreditPair(t *testing.T) {
|
||||
path := writeFile(t, "st.csv", `2026-02-01;RENT;800.00;
|
||||
2026-02-05;SALARY;;2500.00
|
||||
`)
|
||||
acc := &config.Account{Slug: "checking", Currency: "EUR", Parser: "csv", CSV: &config.CSVConfig{
|
||||
Delimiter: ";",
|
||||
Date: config.Column{Col: 0, Layout: "2006-01-02"},
|
||||
Description: config.Column{Col: 1},
|
||||
Debit: &config.Column{Col: 2},
|
||||
Credit: &config.Column{Col: 3},
|
||||
}}
|
||||
p, err := For(acc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := p.Parse(path, acc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("got %d txns, want 2: %+v", len(got), got)
|
||||
}
|
||||
if got[0].AmountMinor != -80000 {
|
||||
t.Errorf("debit row = %d, want -80000", got[0].AmountMinor)
|
||||
}
|
||||
if got[1].AmountMinor != 250000 {
|
||||
t.Errorf("credit row = %d, want 250000", got[1].AmountMinor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVConfigErrors(t *testing.T) {
|
||||
cases := map[string]*config.Account{
|
||||
"no csv section": {Slug: "a", Parser: "csv"},
|
||||
"no amount columns": {Slug: "a", Parser: "csv", CSV: &config.CSVConfig{
|
||||
Date: config.Column{Layout: "2006-01-02"},
|
||||
}},
|
||||
"amount and debit together": {Slug: "a", Parser: "csv", CSV: &config.CSVConfig{
|
||||
Date: config.Column{Layout: "2006-01-02"},
|
||||
Amount: &config.Column{Col: 1},
|
||||
Debit: &config.Column{Col: 2},
|
||||
}},
|
||||
"no date layout": {Slug: "a", Parser: "csv", CSV: &config.CSVConfig{
|
||||
Amount: &config.Column{Col: 1},
|
||||
}},
|
||||
}
|
||||
for name, acc := range cases {
|
||||
if _, err := For(acc); err == nil {
|
||||
t.Errorf("%s: expected an error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownParser(t *testing.T) {
|
||||
if _, err := For(&config.Account{Slug: "a", Parser: "nope"}); err == nil {
|
||||
t.Error("expected an error for an unregistered parser")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register("nlb", func(acc *config.Account) (Parser, error) {
|
||||
return &nlbParser{digits: acc.Digits()}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// nlbParser reads an NLB izpisek PDF.
|
||||
//
|
||||
// A transaction starts on a line beginning with a dd.mm.yy date and ends with
|
||||
// the signed amount and the running balance. Long descriptions and the
|
||||
// counterparty account wrap onto indented continuation lines below.
|
||||
type nlbParser struct {
|
||||
digits int
|
||||
}
|
||||
|
||||
var (
|
||||
// A transaction line starts with a two-digit date.
|
||||
nlbStart = regexp.MustCompile(`^\d{2}\.\d{2}\.\d{2}\s`)
|
||||
// The whole line: date, free-form middle, signed amount, running balance.
|
||||
// Both the ASCII hyphen and the typographic minus U+2212 count as a sign;
|
||||
// which one a PDF carries depends on the font it was produced with.
|
||||
nlbLine = regexp.MustCompile(
|
||||
`^(?P<date>\d{2}\.\d{2}\.\d{2})\s+` +
|
||||
`(?P<middle>.*?)\s+` +
|
||||
`(?P<amount>[-+\x{2212}][\d.,]+)\s+` +
|
||||
`(?P<balance>[-\x{2212}]?[\d.,]+[-\x{2212}]?)\s*$`)
|
||||
// Columns inside a line are separated by four or more spaces.
|
||||
nlbFieldSplit = regexp.MustCompile(`\s{4,}`)
|
||||
// A Slovenian IBAN, which is the counterparty account when present.
|
||||
nlbAccount = regexp.MustCompile(`^SI\d{2}(?:\s?\d{4}){3}\s?\d{3}$`)
|
||||
)
|
||||
|
||||
// nlbMinContinuationIndent is the shallowest indent a wrapped description line
|
||||
// may have. The real threshold is the description column of the transaction
|
||||
// the line belongs to, measured per line rather than hardcoded: pdftotext
|
||||
// squeezes runs of spaces, so absolute columns shift with the font and page
|
||||
// size of the PDF. This floor only rejects flush-left page furniture.
|
||||
const nlbMinContinuationIndent = 2
|
||||
|
||||
// nlbDateLayout is the two-digit-year date NLB prints.
|
||||
const nlbDateLayout = "02.01.06"
|
||||
|
||||
func (p *nlbParser) Parse(path string, acc *config.Account) ([]RawTxn, error) {
|
||||
text, err := pdfToText(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseNLBText(text, p.digits)
|
||||
}
|
||||
|
||||
// parseNLBText holds the whole parser, separated from PDF extraction so it can
|
||||
// be tested against captured pdftotext output.
|
||||
func parseNLBText(text string, digits int) ([]RawTxn, error) {
|
||||
var (
|
||||
txns []RawTxn
|
||||
accts []string // counterparty per transaction, built up alongside
|
||||
descAt int // description column of the last transaction line
|
||||
)
|
||||
|
||||
for _, page := range pages(text) {
|
||||
// Continuation lines may only attach to a transaction started on the
|
||||
// same page, so a wrapped line at the top of a page is page furniture.
|
||||
pageStart := len(txns)
|
||||
|
||||
for i, line := range strings.Split(page, "\n") {
|
||||
if strings.TrimSpace(line) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if nlbStart.MatchString(line) {
|
||||
txn, account, col, err := parseNLBLine(line, digits)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("line %d: %w", i+1, err)
|
||||
}
|
||||
txns = append(txns, txn)
|
||||
accts = append(accts, account)
|
||||
descAt = col
|
||||
continue
|
||||
}
|
||||
|
||||
if len(txns) <= pageStart {
|
||||
continue // header, before any transaction on this page
|
||||
}
|
||||
// A wrapped description sits under the description column of the
|
||||
// transaction it continues; anything to the left of that is a
|
||||
// footer or a column heading.
|
||||
threshold := max(descAt, nlbMinContinuationIndent)
|
||||
if indent(line) < threshold {
|
||||
continue
|
||||
}
|
||||
|
||||
parts := nlbFieldSplit.Split(strings.TrimSpace(line), -1)
|
||||
last := len(txns) - 1
|
||||
txns[last].Description += " " + parts[0]
|
||||
if len(parts) > 1 {
|
||||
accts[last] = strings.TrimSpace(accts[last] + " " + parts[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i := range txns {
|
||||
txns[i].Counterparty = accts[i]
|
||||
txns[i].Description = strings.Join(strings.Fields(txns[i].Description), " ")
|
||||
}
|
||||
return txns, nil
|
||||
}
|
||||
|
||||
// parseNLBLine parses one transaction line, returning the transaction, the
|
||||
// counterparty account found among its columns, and the column at which the
|
||||
// description starts, which is where its wrapped lines will sit.
|
||||
func parseNLBLine(line string, digits int) (RawTxn, string, int, error) {
|
||||
trimmed := strings.TrimRight(line, " \t\r")
|
||||
loc := nlbLine.FindStringSubmatchIndex(trimmed)
|
||||
if loc == nil {
|
||||
return RawTxn{}, "", 0, fmt.Errorf("line starts with a date but has no amount and balance: %q", strings.TrimSpace(line))
|
||||
}
|
||||
group := func(name string) string {
|
||||
i := nlbLine.SubexpIndex(name) * 2
|
||||
if loc[i] < 0 {
|
||||
return ""
|
||||
}
|
||||
return trimmed[loc[i]:loc[i+1]]
|
||||
}
|
||||
var (
|
||||
date = group("date")
|
||||
middle = group("middle")
|
||||
amount = group("amount")
|
||||
balance = group("balance")
|
||||
descCol = loc[nlbLine.SubexpIndex("middle")*2]
|
||||
)
|
||||
|
||||
d, err := time.Parse(nlbDateLayout, date)
|
||||
if err != nil {
|
||||
return RawTxn{}, "", 0, fmt.Errorf("date %q: %w", date, err)
|
||||
}
|
||||
|
||||
amountMinor, err := parseNLBNumber(amount, digits)
|
||||
if err != nil {
|
||||
return RawTxn{}, "", 0, fmt.Errorf("amount %q: %w", amount, err)
|
||||
}
|
||||
balanceMinor, err := parseNLBNumber(balance, digits)
|
||||
if err != nil {
|
||||
return RawTxn{}, "", 0, fmt.Errorf("balance %q: %w", balance, err)
|
||||
}
|
||||
|
||||
// The middle holds the description and, sometimes, the counterparty IBAN.
|
||||
var (
|
||||
account string
|
||||
desc []string
|
||||
)
|
||||
for _, f := range nlbFieldSplit.Split(strings.TrimSpace(middle), -1) {
|
||||
if f == "" {
|
||||
continue
|
||||
}
|
||||
if nlbAccount.MatchString(f) {
|
||||
account = f
|
||||
continue
|
||||
}
|
||||
desc = append(desc, f)
|
||||
}
|
||||
|
||||
return RawTxn{
|
||||
Date: d.Format("2006-01-02"),
|
||||
Description: strings.Join(desc, " "),
|
||||
AmountMinor: amountMinor,
|
||||
BalanceMinor: &balanceMinor,
|
||||
}, account, descCol, nil
|
||||
}
|
||||
|
||||
// parseNLBNumber reads the 1.234,56 format, where a trailing minus marks a
|
||||
// negative balance.
|
||||
func parseNLBNumber(s string, digits int) (int64, error) {
|
||||
return ParseAmount(s, ",", ".", digits)
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package parser
|
||||
|
||||
import "testing"
|
||||
|
||||
// A page as pdftotext -layout renders it: a header, transaction lines ending
|
||||
// in amount and balance, and indented continuation lines carrying wrapped
|
||||
// descriptions and the counterparty IBAN.
|
||||
// The description column sits at 15, which is what the script's
|
||||
// CONTINUATION_INDENT tells us about the real layout.
|
||||
const nlbPage = ` NLB d.d.
|
||||
IZPISEK 001/2026
|
||||
|
||||
DATUM OPIS ZNESEK STANJE
|
||||
|
||||
02.01.26 PLACILO S KARTICO -45,20 1.234,56
|
||||
LIDL SOFIA 4412 SI56 1234 5678 9012 345
|
||||
05.01.26 PRILIV PLACE +2.500,00 3.734,56
|
||||
ACME DOO
|
||||
10.01.26 PRENOS NA VARCEVALNI -500,00 3.234,56
|
||||
15.01.26 NEGATIVNO STANJE -3.300,00 65,44-
|
||||
|
||||
Stran 1
|
||||
`
|
||||
|
||||
func TestParseNLBText(t *testing.T) {
|
||||
txns, err := parseNLBText(nlbPage, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 4 {
|
||||
t.Fatalf("got %d transactions, want 4: %+v", len(txns), txns)
|
||||
}
|
||||
|
||||
first := txns[0]
|
||||
if first.Date != "2026-01-02" {
|
||||
t.Errorf("date = %q, want 2026-01-02", first.Date)
|
||||
}
|
||||
// The continuation line is appended to the description.
|
||||
if first.Description != "PLACILO S KARTICO LIDL SOFIA 4412" {
|
||||
t.Errorf("description = %q", first.Description)
|
||||
}
|
||||
// ...and the IBAN in its second column becomes the counterparty.
|
||||
if first.Counterparty != "SI56 1234 5678 9012 345" {
|
||||
t.Errorf("counterparty = %q", first.Counterparty)
|
||||
}
|
||||
if first.AmountMinor != -4520 {
|
||||
t.Errorf("amount = %d, want -4520", first.AmountMinor)
|
||||
}
|
||||
if first.BalanceMinor == nil || *first.BalanceMinor != 123456 {
|
||||
t.Errorf("balance = %v, want 123456", first.BalanceMinor)
|
||||
}
|
||||
|
||||
if txns[1].AmountMinor != 250000 {
|
||||
t.Errorf("credit amount = %d, want 250000", txns[1].AmountMinor)
|
||||
}
|
||||
if txns[1].Description != "PRILIV PLACE ACME DOO" {
|
||||
t.Errorf("description = %q", txns[1].Description)
|
||||
}
|
||||
|
||||
// A transaction with no continuation line keeps an empty counterparty.
|
||||
if txns[2].Counterparty != "" {
|
||||
t.Errorf("counterparty = %q, want empty", txns[2].Counterparty)
|
||||
}
|
||||
|
||||
// A trailing minus marks a negative balance.
|
||||
if txns[3].BalanceMinor == nil || *txns[3].BalanceMinor != -6544 {
|
||||
t.Errorf("balance = %v, want -6544", txns[3].BalanceMinor)
|
||||
}
|
||||
}
|
||||
|
||||
// A wrapped line at the top of a page must not attach to the last transaction
|
||||
// of the previous page.
|
||||
func TestParseNLBPageBoundary(t *testing.T) {
|
||||
text := "02.01.26 FIRST -10,00 100,00\n" +
|
||||
"\f" +
|
||||
" STRAY CONTINUATION AT TOP OF PAGE\n" +
|
||||
"03.01.26 SECOND -20,00 80,00\n" +
|
||||
" REAL CONTINUATION\n"
|
||||
|
||||
txns, err := parseNLBText(text, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 2 {
|
||||
t.Fatalf("got %d transactions, want 2", len(txns))
|
||||
}
|
||||
if txns[0].Description != "FIRST" {
|
||||
t.Errorf("first description = %q, want FIRST (page furniture must not leak in)", txns[0].Description)
|
||||
}
|
||||
if txns[1].Description != "SECOND REAL CONTINUATION" {
|
||||
t.Errorf("second description = %q", txns[1].Description)
|
||||
}
|
||||
}
|
||||
|
||||
// Shallowly indented lines are footers, not wrapped descriptions.
|
||||
func TestParseNLBIgnoresShallowIndent(t *testing.T) {
|
||||
text := "02.01.26 FIRST -10,00 100,00\n" +
|
||||
" Stran 1 od 2\n"
|
||||
|
||||
txns, err := parseNLBText(text, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 1 || txns[0].Description != "FIRST" {
|
||||
t.Errorf("got %+v, want a single FIRST transaction", txns)
|
||||
}
|
||||
}
|
||||
|
||||
// A date-led line without an amount and balance is a real problem, not
|
||||
// something to silently drop.
|
||||
func TestParseNLBRejectsMalformedLine(t *testing.T) {
|
||||
text := "02.01.26 NO AMOUNT OR BALANCE HERE\n"
|
||||
if _, err := parseNLBText(text, 2); err == nil {
|
||||
t.Error("expected an error for a transaction line with no amount")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
// Package parser turns a statement file into raw transactions.
|
||||
//
|
||||
// Every bank needs its own extraction logic, so parsers are looked up by name
|
||||
// from a registry. Two are built in: "csv" for delimited exports with
|
||||
// configurable columns, and "cmd" for shelling out to an external extractor
|
||||
// (which is how the existing Python scripts are used until they are ported).
|
||||
// Ported extractors register themselves here and become usable by putting
|
||||
// their name in an account.toml.
|
||||
package parser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
)
|
||||
|
||||
// RawTxn is one line as it came out of a statement, before fingerprinting,
|
||||
// deduplication or tagging.
|
||||
type RawTxn struct {
|
||||
Date string // YYYY-MM-DD
|
||||
Description string
|
||||
AmountMinor int64 // signed; negative is an outflow
|
||||
|
||||
// Counterparty is the other side's account number, where the statement
|
||||
// gives one. Optional.
|
||||
Counterparty string
|
||||
// Type is the bank's own classification of the transaction. Optional.
|
||||
Type string
|
||||
// BalanceMinor is the running balance after this transaction, when the
|
||||
// statement reports one. Optional; enables the balance-chain check.
|
||||
BalanceMinor *int64
|
||||
}
|
||||
|
||||
// Parser extracts transactions from a single statement file.
|
||||
type Parser interface {
|
||||
// Parse reads the statement at path. digits is the account's minor-unit
|
||||
// scale, so parsers can convert decimal strings without guessing.
|
||||
Parse(path string, acc *config.Account) ([]RawTxn, error)
|
||||
}
|
||||
|
||||
// Warner is an optional interface for parsers that legitimately drop rows --
|
||||
// pending transactions, other currencies -- and want to say so. The importer
|
||||
// collects the warnings from the most recent Parse call and reports them.
|
||||
type Warner interface {
|
||||
Warnings() []string
|
||||
}
|
||||
|
||||
// Factory builds a parser from an account's config, validating it up front so
|
||||
// a bad account.toml fails before any file is read.
|
||||
type Factory func(acc *config.Account) (Parser, error)
|
||||
|
||||
var (
|
||||
mu sync.RWMutex
|
||||
registry = map[string]Factory{}
|
||||
)
|
||||
|
||||
// Register adds a named parser. It panics on a duplicate name, since that can
|
||||
// only be a programming error at init time.
|
||||
func Register(name string, f Factory) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if _, dup := registry[name]; dup {
|
||||
panic("parser: duplicate registration of " + name)
|
||||
}
|
||||
registry[name] = f
|
||||
}
|
||||
|
||||
// For builds the parser named by acc.Parser.
|
||||
func For(acc *config.Account) (Parser, error) {
|
||||
mu.RLock()
|
||||
f, ok := registry[acc.Parser]
|
||||
mu.RUnlock()
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("account %s: unknown parser %q (available: %v)", acc.Slug, acc.Parser, Names())
|
||||
}
|
||||
return f(acc)
|
||||
}
|
||||
|
||||
// Names lists the registered parsers, for error messages and `money parsers`.
|
||||
func Names() []string {
|
||||
mu.RLock()
|
||||
defer mu.RUnlock()
|
||||
out := make([]string, 0, len(registry))
|
||||
for name := range registry {
|
||||
out = append(out, name)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// pdfToTextTimeout bounds extraction so a malformed PDF cannot wedge an import.
|
||||
const pdfToTextTimeout = 2 * time.Minute
|
||||
|
||||
// pdfToText renders a PDF as text with the original column positions
|
||||
// preserved, which is what the statement parsers key off.
|
||||
//
|
||||
// This shells out to poppler's pdftotext rather than decoding the PDF in Go:
|
||||
// the layout reconstruction it does is the whole reason the column-based
|
||||
// parsers work, and no Go library matches it.
|
||||
func pdfToText(path string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), pdfToTextTimeout)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, "pdftotext", "-layout", path, "-")
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return "", fmt.Errorf("pdftotext timed out after %s on %s", pdfToTextTimeout, path)
|
||||
}
|
||||
if errors := strings.TrimSpace(stderr.String()); errors != "" {
|
||||
return "", fmt.Errorf("pdftotext %s: %w: %s", path, err, errors)
|
||||
}
|
||||
if _, lookErr := exec.LookPath("pdftotext"); lookErr != nil {
|
||||
return "", fmt.Errorf("pdftotext is not installed (it ships with poppler-utils): %w", lookErr)
|
||||
}
|
||||
return "", fmt.Errorf("pdftotext %s: %w", path, err)
|
||||
}
|
||||
return stdout.String(), nil
|
||||
}
|
||||
|
||||
// pages splits pdftotext output on form feeds.
|
||||
func pages(text string) []string {
|
||||
return strings.Split(text, "\f")
|
||||
}
|
||||
|
||||
// indent counts the leading spaces of a line.
|
||||
func indent(line string) int {
|
||||
return len(line) - len(strings.TrimLeft(line, " "))
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register("revolut", func(acc *config.Account) (Parser, error) {
|
||||
return &revolutParser{digits: acc.Digits(), currency: strings.ToUpper(acc.Currency)}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// revolutParser reads a Revolut account-statement CSV export.
|
||||
//
|
||||
// Revolut books the fee alongside the transaction rather than as its own line,
|
||||
// and reports pending transactions that have no balance yet. Both are handled
|
||||
// the way the original extraction script did: the fee is folded into the
|
||||
// amount, and anything not COMPLETED is skipped.
|
||||
//
|
||||
// A single export can mix currencies. Since an account here has one currency,
|
||||
// rows in others are skipped and reported; importing them is a matter of
|
||||
// giving that currency its own account folder.
|
||||
type revolutParser struct {
|
||||
digits int
|
||||
currency string
|
||||
warnings []string
|
||||
}
|
||||
|
||||
// revolutIBAN finds a counterparty account inside a description.
|
||||
var revolutIBAN = regexp.MustCompile(`\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b`)
|
||||
|
||||
// Columns the parser needs; a missing one is a hard error rather than a
|
||||
// silently empty field.
|
||||
var revolutColumns = []string{
|
||||
"Type", "Completed Date", "Description", "Amount", "Fee", "Currency", "State", "Balance",
|
||||
}
|
||||
|
||||
func (p *revolutParser) Warnings() []string { return p.warnings }
|
||||
|
||||
func (p *revolutParser) Parse(path string, acc *config.Account) ([]RawTxn, error) {
|
||||
p.warnings = nil
|
||||
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
r := csv.NewReader(f)
|
||||
r.FieldsPerRecord = -1
|
||||
r.LazyQuotes = true
|
||||
|
||||
header, err := r.Read()
|
||||
if err == io.EOF {
|
||||
return nil, fmt.Errorf("%s is empty", path)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
index, err := revolutHeaderIndex(header)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
|
||||
type record struct {
|
||||
fields []string
|
||||
line int
|
||||
}
|
||||
var records []record
|
||||
for line := 2; ; line++ {
|
||||
rec, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: row %d: %w", path, line, err)
|
||||
}
|
||||
if isBlank(rec) {
|
||||
continue
|
||||
}
|
||||
records = append(records, record{fields: rec, line: line})
|
||||
}
|
||||
|
||||
// The export is not in date order, and the running balance only makes
|
||||
// sense along one. A stable sort also keeps identical rows in file order,
|
||||
// which the importer's deduplication relies on.
|
||||
sort.SliceStable(records, func(i, j int) bool {
|
||||
return index.get(records[i].fields, "Completed Date") < index.get(records[j].fields, "Completed Date")
|
||||
})
|
||||
|
||||
var (
|
||||
txns []RawTxn
|
||||
skipped = map[string]int{}
|
||||
currencies = map[string]int{}
|
||||
)
|
||||
for _, rec := range records {
|
||||
get := func(name string) string { return index.get(rec.fields, name) }
|
||||
|
||||
if state := get("State"); state != "COMPLETED" || get("Balance") == "" {
|
||||
if state == "" {
|
||||
state = "unfinished"
|
||||
}
|
||||
skipped[state]++
|
||||
continue
|
||||
}
|
||||
if currency := strings.ToUpper(get("Currency")); currency != p.currency {
|
||||
currencies[currency]++
|
||||
continue
|
||||
}
|
||||
|
||||
txn, err := p.row(get, p.digits)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: row %d: %w", path, rec.line, err)
|
||||
}
|
||||
txns = append(txns, txn)
|
||||
}
|
||||
|
||||
for _, state := range sortedKeys(skipped) {
|
||||
p.warnings = append(p.warnings, fmt.Sprintf("skipped %d %s transactions", skipped[state], state))
|
||||
}
|
||||
for _, currency := range sortedKeys(currencies) {
|
||||
p.warnings = append(p.warnings, fmt.Sprintf(
|
||||
"skipped %d rows in %s; give that currency its own account folder to import them",
|
||||
currencies[currency], currency))
|
||||
}
|
||||
return txns, nil
|
||||
}
|
||||
|
||||
func (p *revolutParser) row(get func(string) string, digits int) (RawTxn, error) {
|
||||
d, err := time.Parse("2006-01-02", firstN(get("Completed Date"), 10))
|
||||
if err != nil {
|
||||
return RawTxn{}, fmt.Errorf("completed date %q: %w", get("Completed Date"), err)
|
||||
}
|
||||
|
||||
amount, err := ParseAmount(get("Amount"), ".", "", digits)
|
||||
if err != nil {
|
||||
return RawTxn{}, fmt.Errorf("amount: %w", err)
|
||||
}
|
||||
var fee int64
|
||||
if raw := strings.TrimSpace(get("Fee")); raw != "" {
|
||||
if fee, err = ParseAmount(raw, ".", "", digits); err != nil {
|
||||
return RawTxn{}, fmt.Errorf("fee: %w", err)
|
||||
}
|
||||
}
|
||||
balance, err := ParseAmount(get("Balance"), ".", "", digits)
|
||||
if err != nil {
|
||||
return RawTxn{}, fmt.Errorf("balance: %w", err)
|
||||
}
|
||||
|
||||
// The fee is deducted along with the amount rather than booked separately,
|
||||
// so it has to be folded in for the balance chain to hold.
|
||||
desc := strings.TrimSpace(get("Description"))
|
||||
if fee != 0 {
|
||||
desc += fmt.Sprintf(" (fee %s)", model.FormatMinor(fee, digits))
|
||||
}
|
||||
|
||||
counterparty := revolutIBAN.FindString(desc)
|
||||
|
||||
return RawTxn{
|
||||
Date: d.Format("2006-01-02"),
|
||||
Description: desc,
|
||||
AmountMinor: amount - fee,
|
||||
Counterparty: counterparty,
|
||||
Type: strings.TrimSpace(get("Type")),
|
||||
BalanceMinor: &balance,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// headerIndex maps a Revolut column name to its position.
|
||||
type headerIndex map[string]int
|
||||
|
||||
func (h headerIndex) get(fields []string, name string) string {
|
||||
i, ok := h[name]
|
||||
if !ok || i >= len(fields) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(fields[i])
|
||||
}
|
||||
|
||||
func revolutHeaderIndex(header []string) (headerIndex, error) {
|
||||
index := headerIndex{}
|
||||
for i, name := range header {
|
||||
index[strings.TrimSpace(name)] = i
|
||||
}
|
||||
var missing []string
|
||||
for _, name := range revolutColumns {
|
||||
if _, ok := index[name]; !ok {
|
||||
missing = append(missing, name)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return nil, fmt.Errorf("statement is missing the %s column(s); header was %v",
|
||||
strings.Join(missing, ", "), header)
|
||||
}
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]int) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func firstN(s string, n int) string {
|
||||
if len(s) < n {
|
||||
return s
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
)
|
||||
|
||||
// A Revolut export: out of date order, mixed currencies, a pending row, a row
|
||||
// with a fee, and a transfer carrying an IBAN.
|
||||
const revolutCSV = `Type,Product,Started Date,Completed Date,Description,Amount,Fee,Currency,State,Balance
|
||||
CARD_PAYMENT,Current,2026-01-06 09:12:00,2026-01-06 09:12:00,LIDL SOFIA 4412,-45.20,0.00,EUR,COMPLETED,1234.56
|
||||
TOPUP,Current,2026-01-05 08:00:00,2026-01-05 08:00:00,Payment from ACME,2500.00,0.00,EUR,COMPLETED,1279.76
|
||||
TRANSFER,Current,2026-01-10 10:00:00,2026-01-10 10:00:00,To savings SI56123456789012345,-500.00,0.35,EUR,COMPLETED,734.41
|
||||
CARD_PAYMENT,Current,2026-01-11 10:00:00,,Pending coffee,-3.50,0.00,EUR,PENDING,
|
||||
EXCHANGE,Current,2026-01-12 10:00:00,2026-01-12 10:00:00,Tokyo hotel,-15000,0,JPY,COMPLETED,50000
|
||||
`
|
||||
|
||||
func revolutAccount(currency string, digits int) *config.Account {
|
||||
return &config.Account{Slug: "revolut", Currency: currency, MinorDigits: &digits, Parser: "revolut"}
|
||||
}
|
||||
|
||||
func TestRevolutParse(t *testing.T) {
|
||||
path := writeFile(t, "account-statement.csv", revolutCSV)
|
||||
acc := revolutAccount("EUR", 2)
|
||||
|
||||
p, err := For(acc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
txns, err := p.Parse(path, acc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 3 {
|
||||
t.Fatalf("got %d transactions, want 3: %+v", len(txns), txns)
|
||||
}
|
||||
|
||||
// Rows come back in completed-date order, not file order.
|
||||
wantDates := []string{"2026-01-05", "2026-01-06", "2026-01-10"}
|
||||
for i, want := range wantDates {
|
||||
if txns[i].Date != want {
|
||||
t.Errorf("txn %d date = %q, want %q", i, txns[i].Date, want)
|
||||
}
|
||||
}
|
||||
|
||||
if txns[0].Type != "TOPUP" {
|
||||
t.Errorf("type = %q, want TOPUP", txns[0].Type)
|
||||
}
|
||||
if txns[1].AmountMinor != -4520 {
|
||||
t.Errorf("card payment = %d, want -4520", txns[1].AmountMinor)
|
||||
}
|
||||
|
||||
// The fee is folded into the amount and noted in the description.
|
||||
transfer := txns[2]
|
||||
if transfer.AmountMinor != -50035 {
|
||||
t.Errorf("transfer with fee = %d, want -50035 (amount minus fee)", transfer.AmountMinor)
|
||||
}
|
||||
if !strings.Contains(transfer.Description, "(fee 0.35)") {
|
||||
t.Errorf("description = %q, want a fee note", transfer.Description)
|
||||
}
|
||||
if transfer.Counterparty != "SI56123456789012345" {
|
||||
t.Errorf("counterparty = %q, want the IBAN from the description", transfer.Counterparty)
|
||||
}
|
||||
if transfer.BalanceMinor == nil || *transfer.BalanceMinor != 73441 {
|
||||
t.Errorf("balance = %v, want 73441", transfer.BalanceMinor)
|
||||
}
|
||||
|
||||
// Pending and foreign-currency rows are skipped, and said so.
|
||||
warnings := strings.Join(p.(Warner).Warnings(), "\n")
|
||||
if !strings.Contains(warnings, "1 PENDING") {
|
||||
t.Errorf("warnings = %q, want a note about the pending row", warnings)
|
||||
}
|
||||
if !strings.Contains(warnings, "JPY") {
|
||||
t.Errorf("warnings = %q, want a note about the JPY row", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
// The same export read as a JPY account picks up the row the EUR account skipped.
|
||||
func TestRevolutOtherCurrencyViaSecondAccount(t *testing.T) {
|
||||
path := writeFile(t, "account-statement.csv", revolutCSV)
|
||||
acc := revolutAccount("JPY", 0)
|
||||
|
||||
p, err := For(acc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
txns, err := p.Parse(path, acc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 1 {
|
||||
t.Fatalf("got %d transactions, want 1: %+v", len(txns), txns)
|
||||
}
|
||||
// JPY has no minor unit, so the amount is whole yen.
|
||||
if txns[0].AmountMinor != -15000 {
|
||||
t.Errorf("amount = %d, want -15000", txns[0].AmountMinor)
|
||||
}
|
||||
if txns[0].BalanceMinor == nil || *txns[0].BalanceMinor != 50000 {
|
||||
t.Errorf("balance = %v, want 50000", txns[0].BalanceMinor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRevolutMissingColumn(t *testing.T) {
|
||||
path := writeFile(t, "bad.csv", "Type,Description,Amount\nCARD_PAYMENT,X,-1.00\n")
|
||||
acc := revolutAccount("EUR", 2)
|
||||
p, err := For(acc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = p.Parse(path, acc)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error for a statement missing required columns")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "Balance") {
|
||||
t.Errorf("error = %v, want it to name the missing columns", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Register("traderepublic", func(acc *config.Account) (Parser, error) {
|
||||
return &tradeRepublicParser{digits: acc.Digits()}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// tradeRepublicParser reads a Trade Republic account statement PDF.
|
||||
//
|
||||
// The layout varies between statements: a transaction may sit entirely on one
|
||||
// line, or have its date, type and description stacked across several. Rather
|
||||
// than guess, every token is assigned to whichever column heading its start
|
||||
// position is closest to, which handles both shapes.
|
||||
type tradeRepublicParser struct {
|
||||
digits int
|
||||
}
|
||||
|
||||
var (
|
||||
trHeader = regexp.MustCompile(`^\s*DATE\b.*\bMONEY IN\b.*\bMONEY OUT\b.*\bBALANCE\b`)
|
||||
// The sign may be an ASCII hyphen or a typographic minus, depending on the
|
||||
// font the PDF was produced with.
|
||||
trAmount = regexp.MustCompile(`[-\x{2212}]?€\s?[-\x{2212}]?[\d,]+\.\d{2}`)
|
||||
trToken = regexp.MustCompile(`\S+`)
|
||||
trFullDate = regexp.MustCompile(`^\d{2} [A-Z][a-z]{2} \d{4}$`)
|
||||
trIBAN = regexp.MustCompile(`\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b`)
|
||||
)
|
||||
|
||||
var (
|
||||
trTextColumns = []string{"DATE", "TYPE", "DESCRIPTION"}
|
||||
trMoneyColumns = []string{"MONEY IN", "MONEY OUT", "BALANCE"}
|
||||
)
|
||||
|
||||
// trAmountSlack lets an amount sit slightly left of the MONEY IN column
|
||||
// without being mistaken for description text.
|
||||
const trAmountSlack = 5
|
||||
|
||||
// trDateLayout is the date Trade Republic prints, e.g. "05 Jan 2026".
|
||||
const trDateLayout = "02 Jan 2006"
|
||||
|
||||
func (p *tradeRepublicParser) Parse(path string, acc *config.Account) ([]RawTxn, error) {
|
||||
text, err := pdfToText(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseTradeRepublicText(text, p.digits)
|
||||
}
|
||||
|
||||
// parseTradeRepublicText holds the whole parser, separated from PDF extraction
|
||||
// so it can be tested against captured pdftotext output.
|
||||
func parseTradeRepublicText(text string, digits int) ([]RawTxn, error) {
|
||||
var txns []RawTxn
|
||||
|
||||
for _, page := range pages(text) {
|
||||
lines := strings.Split(page, "\n")
|
||||
header := -1
|
||||
for i, line := range lines {
|
||||
if trHeader.MatchString(line) {
|
||||
header = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if header < 0 {
|
||||
continue // a cover page or disclaimer, with no transaction table
|
||||
}
|
||||
|
||||
cols, err := trColumnStarts(lines[header])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, block := range trBlocks(lines[header+1:]) {
|
||||
txn, ok, err := parseTradeRepublicBlock(block, cols, digits)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok {
|
||||
txns = append(txns, txn)
|
||||
}
|
||||
}
|
||||
}
|
||||
return txns, nil
|
||||
}
|
||||
|
||||
// trColumnStarts records where each heading begins on the header line; those
|
||||
// positions are what every token is measured against.
|
||||
func trColumnStarts(header string) (map[string]int, error) {
|
||||
cols := map[string]int{}
|
||||
for _, name := range append(append([]string{}, trTextColumns...), trMoneyColumns...) {
|
||||
i := strings.Index(header, name)
|
||||
if i < 0 {
|
||||
return nil, fmt.Errorf("statement header has no %q column: %q", name, strings.TrimSpace(header))
|
||||
}
|
||||
cols[name] = i
|
||||
}
|
||||
return cols, nil
|
||||
}
|
||||
|
||||
// trBlocks groups the consecutive non-blank lines that make up one transaction.
|
||||
func trBlocks(lines []string) [][]string {
|
||||
var (
|
||||
out [][]string
|
||||
block []string
|
||||
)
|
||||
for _, line := range lines {
|
||||
if strings.TrimSpace(line) != "" {
|
||||
block = append(block, line)
|
||||
continue
|
||||
}
|
||||
if len(block) > 0 {
|
||||
out = append(out, block)
|
||||
block = nil
|
||||
}
|
||||
}
|
||||
if len(block) > 0 {
|
||||
out = append(out, block)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// nearest returns the column whose start is closest to pos.
|
||||
func nearest(pos int, cols map[string]int, names []string) string {
|
||||
best, bestDist := "", -1
|
||||
for _, name := range names {
|
||||
d := pos - cols[name]
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
if bestDist < 0 || d < bestDist {
|
||||
best, bestDist = name, d
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func parseTradeRepublicBlock(lines []string, cols map[string]int, digits int) (RawTxn, bool, error) {
|
||||
words := map[string][]string{}
|
||||
amounts := map[string]*int64{}
|
||||
var descChunks []string
|
||||
|
||||
for _, line := range lines {
|
||||
// Amounts are found first: everything to their left is text, and the
|
||||
// cut keeps them from being read as description tokens.
|
||||
cut := len(line)
|
||||
for _, loc := range trAmount.FindAllStringIndex(line, -1) {
|
||||
if loc[0] < cols["MONEY IN"]-trAmountSlack {
|
||||
continue // a figure inside the description, not a money column
|
||||
}
|
||||
if loc[0] < cut {
|
||||
cut = loc[0]
|
||||
}
|
||||
v, err := parseTradeRepublicNumber(line[loc[0]:loc[1]], digits)
|
||||
if err != nil {
|
||||
return RawTxn{}, false, fmt.Errorf("amount %q: %w", line[loc[0]:loc[1]], err)
|
||||
}
|
||||
amounts[nearest(loc[0], cols, trMoneyColumns)] = &v
|
||||
}
|
||||
|
||||
// A description fragment per line, so wrapped text can be rejoined.
|
||||
var lineDesc []string
|
||||
for _, loc := range trToken.FindAllStringIndex(line[:cut], -1) {
|
||||
column := nearest(loc[0], cols, trTextColumns)
|
||||
token := line[loc[0]:loc[1]]
|
||||
if column == "DESCRIPTION" {
|
||||
lineDesc = append(lineDesc, token)
|
||||
continue
|
||||
}
|
||||
words[column] = append(words[column], token)
|
||||
}
|
||||
if len(lineDesc) > 0 {
|
||||
descChunks = append(descChunks, strings.Join(lineDesc, " "))
|
||||
}
|
||||
}
|
||||
|
||||
// A block without a full date and a balance is a heading or a footer.
|
||||
date := strings.Join(words["DATE"], " ")
|
||||
if !trFullDate.MatchString(date) || amounts["BALANCE"] == nil {
|
||||
return RawTxn{}, false, nil
|
||||
}
|
||||
d, err := time.Parse(trDateLayout, date)
|
||||
if err != nil {
|
||||
return RawTxn{}, false, fmt.Errorf("date %q: %w", date, err)
|
||||
}
|
||||
|
||||
desc := trJoinWrapped(descChunks)
|
||||
amount := deref(amounts["MONEY IN"]) - deref(amounts["MONEY OUT"])
|
||||
|
||||
return RawTxn{
|
||||
Date: d.Format("2006-01-02"),
|
||||
Description: desc,
|
||||
AmountMinor: amount,
|
||||
Counterparty: trIBAN.FindString(desc),
|
||||
Type: strings.Join(words["TYPE"], " "),
|
||||
BalanceMinor: amounts["BALANCE"],
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
// trJoinWrapped glues description fragments split across lines. A fragment
|
||||
// ending in a hyphen was broken mid-word, so it joins without a space.
|
||||
func trJoinWrapped(chunks []string) string {
|
||||
var out string
|
||||
for _, chunk := range chunks {
|
||||
switch {
|
||||
case out == "":
|
||||
out = chunk
|
||||
case len(out) > 1 && strings.HasSuffix(out, "-") && !strings.HasSuffix(out, " -"):
|
||||
out += chunk
|
||||
default:
|
||||
out += " " + chunk
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseTradeRepublicNumber reads the €1,234.56 format.
|
||||
func parseTradeRepublicNumber(s string, digits int) (int64, error) {
|
||||
return ParseAmount(strings.ReplaceAll(s, "€", ""), ".", ",", digits)
|
||||
}
|
||||
|
||||
func deref(v *int64) int64 {
|
||||
if v == nil {
|
||||
return 0
|
||||
}
|
||||
return *v
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// place builds a fixed-width line by putting each string at a given column,
|
||||
// which keeps these fixtures honest about the layout the parser measures.
|
||||
func place(cells map[int]string) string {
|
||||
width := 0
|
||||
for col, text := range cells {
|
||||
if end := col + len(text); end > width {
|
||||
width = end
|
||||
}
|
||||
}
|
||||
line := []byte(strings.Repeat(" ", width))
|
||||
for col, text := range cells {
|
||||
copy(line[col:], text)
|
||||
}
|
||||
return string(line)
|
||||
}
|
||||
|
||||
// Column starts, spaced as a real statement is: wide enough that every token
|
||||
// of "05 Jan 2026" stays closest to DATE rather than drifting into TYPE.
|
||||
const (
|
||||
colDate = 2
|
||||
colType = 20
|
||||
colDesc = 40
|
||||
colMoneyIn = 75
|
||||
colMoneyOut = 90
|
||||
colBalance = 105
|
||||
)
|
||||
|
||||
func trHeaderLine() string {
|
||||
return place(map[int]string{
|
||||
colDate: "DATE", colType: "TYPE", colDesc: "DESCRIPTION",
|
||||
colMoneyIn: "MONEY IN", colMoneyOut: "MONEY OUT", colBalance: "BALANCE",
|
||||
})
|
||||
}
|
||||
|
||||
// A statement mixing both layouts: one transaction on a single line, one with
|
||||
// its fields stacked across three, and one whose description wraps.
|
||||
func trStatement() string {
|
||||
lines := []string{
|
||||
" Trade Republic Bank GmbH",
|
||||
"",
|
||||
trHeaderLine(),
|
||||
"",
|
||||
// Everything on one line.
|
||||
place(map[int]string{
|
||||
colDate: "05 Jan 2026", colType: "Deposit", colDesc: "Payment from ACME",
|
||||
colMoneyIn: "€2,500.00", colBalance: "€2,500.00",
|
||||
}),
|
||||
"",
|
||||
// Stacked across three lines.
|
||||
place(map[int]string{
|
||||
colDate: "06 Jan 2026", colMoneyOut: "€45.20", colBalance: "€2,454.80",
|
||||
}),
|
||||
place(map[int]string{colType: "Card"}),
|
||||
place(map[int]string{colDesc: "LIDL SOFIA 4412"}),
|
||||
"",
|
||||
// Wrapped description, hyphen-broken on the first line, plus an IBAN.
|
||||
place(map[int]string{
|
||||
colDate: "10 Jan 2026", colType: "Transfer", colDesc: "Standing order to sav-",
|
||||
colMoneyOut: "€500.00", colBalance: "€1,954.80",
|
||||
}),
|
||||
place(map[int]string{colDesc: "ings SI56123456789012345"}),
|
||||
"",
|
||||
" Page 1 of 2",
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func TestParseTradeRepublicText(t *testing.T) {
|
||||
txns, err := parseTradeRepublicText(trStatement(), 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 3 {
|
||||
t.Fatalf("got %d transactions, want 3: %+v", len(txns), txns)
|
||||
}
|
||||
|
||||
deposit := txns[0]
|
||||
if deposit.Date != "2026-01-05" {
|
||||
t.Errorf("date = %q, want 2026-01-05", deposit.Date)
|
||||
}
|
||||
if deposit.AmountMinor != 250000 {
|
||||
t.Errorf("money in = %d, want 250000", deposit.AmountMinor)
|
||||
}
|
||||
if deposit.Type != "Deposit" {
|
||||
t.Errorf("type = %q, want Deposit", deposit.Type)
|
||||
}
|
||||
if deposit.Description != "Payment from ACME" {
|
||||
t.Errorf("description = %q", deposit.Description)
|
||||
}
|
||||
|
||||
// The stacked layout must produce the same shape as the single-line one.
|
||||
card := txns[1]
|
||||
if card.Date != "2026-01-06" {
|
||||
t.Errorf("date = %q, want 2026-01-06", card.Date)
|
||||
}
|
||||
if card.AmountMinor != -4520 {
|
||||
t.Errorf("money out = %d, want -4520", card.AmountMinor)
|
||||
}
|
||||
if card.Type != "Card" {
|
||||
t.Errorf("type = %q, want Card", card.Type)
|
||||
}
|
||||
if card.Description != "LIDL SOFIA 4412" {
|
||||
t.Errorf("description = %q", card.Description)
|
||||
}
|
||||
if card.BalanceMinor == nil || *card.BalanceMinor != 245480 {
|
||||
t.Errorf("balance = %v, want 245480", card.BalanceMinor)
|
||||
}
|
||||
|
||||
// A hyphen-broken word rejoins without a space. The hyphen itself is kept,
|
||||
// as the original script does, because descriptions contain real hyphens
|
||||
// that must not be swallowed.
|
||||
transfer := txns[2]
|
||||
if transfer.Description != "Standing order to sav-ings SI56123456789012345" {
|
||||
t.Errorf("description = %q, want the wrapped fragment joined without a space", transfer.Description)
|
||||
}
|
||||
if transfer.Counterparty != "SI56123456789012345" {
|
||||
t.Errorf("counterparty = %q", transfer.Counterparty)
|
||||
}
|
||||
if transfer.AmountMinor != -50000 {
|
||||
t.Errorf("amount = %d, want -50000", transfer.AmountMinor)
|
||||
}
|
||||
}
|
||||
|
||||
// Pages with no transaction table (cover pages, disclaimers) are skipped.
|
||||
func TestParseTradeRepublicSkipsPagesWithoutTable(t *testing.T) {
|
||||
text := " Some cover page with no table at all\n" + "\f" + trStatement()
|
||||
txns, err := parseTradeRepublicText(text, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 3 {
|
||||
t.Errorf("got %d transactions, want 3", len(txns))
|
||||
}
|
||||
}
|
||||
|
||||
// A figure inside the description must not be mistaken for a money column.
|
||||
func TestParseTradeRepublicIgnoresAmountsInDescription(t *testing.T) {
|
||||
text := strings.Join([]string{
|
||||
trHeaderLine(),
|
||||
"",
|
||||
place(map[int]string{
|
||||
colDate: "05 Jan 2026", colType: "Card", colDesc: "Refund of €12.00 order",
|
||||
colMoneyIn: "€12.00", colBalance: "€100.00",
|
||||
}),
|
||||
}, "\n")
|
||||
|
||||
txns, err := parseTradeRepublicText(text, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 1 {
|
||||
t.Fatalf("got %d transactions, want 1", len(txns))
|
||||
}
|
||||
if txns[0].AmountMinor != 1200 {
|
||||
t.Errorf("amount = %d, want 1200", txns[0].AmountMinor)
|
||||
}
|
||||
if !strings.Contains(txns[0].Description, "€12.00") {
|
||||
t.Errorf("description = %q, want the inline figure kept", txns[0].Description)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
// Package report aggregates transactions for the summary views.
|
||||
//
|
||||
// Transfers between the user's own accounts are excluded everywhere: moving
|
||||
// money from checking to savings is not income and not spending.
|
||||
package report
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
)
|
||||
|
||||
// TagTotal is one row of a spending breakdown.
|
||||
type TagTotal struct {
|
||||
Tag string
|
||||
Currency string
|
||||
In int64 // sum of inflows, positive
|
||||
Out int64 // sum of outflows, positive
|
||||
Count int
|
||||
Digits int
|
||||
}
|
||||
|
||||
// Net is inflow minus outflow.
|
||||
func (t TagTotal) Net() int64 { return t.In - t.Out }
|
||||
|
||||
// ByTag groups transactions by currency and effective tag. Untagged rows are
|
||||
// collected under "(untagged)" so they stay visible instead of vanishing.
|
||||
const Untagged = "(untagged)"
|
||||
|
||||
// ByTag returns totals sorted by currency, then by largest outflow first.
|
||||
func ByTag(txns []model.Transaction) []TagTotal {
|
||||
type key struct{ currency, tag string }
|
||||
acc := map[key]*TagTotal{}
|
||||
|
||||
for _, t := range txns {
|
||||
if t.IsTransfer() {
|
||||
continue
|
||||
}
|
||||
tag := t.Tag()
|
||||
if tag == "" {
|
||||
tag = Untagged
|
||||
}
|
||||
k := key{t.Currency, tag}
|
||||
row, ok := acc[k]
|
||||
if !ok {
|
||||
row = &TagTotal{Tag: tag, Currency: t.Currency, Digits: t.MinorDigits}
|
||||
acc[k] = row
|
||||
}
|
||||
if t.AmountMinor < 0 {
|
||||
row.Out += -t.AmountMinor
|
||||
} else {
|
||||
row.In += t.AmountMinor
|
||||
}
|
||||
row.Count++
|
||||
}
|
||||
|
||||
out := make([]TagTotal, 0, len(acc))
|
||||
for _, row := range acc {
|
||||
out = append(out, *row)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Currency != out[j].Currency {
|
||||
return out[i].Currency < out[j].Currency
|
||||
}
|
||||
if out[i].Out != out[j].Out {
|
||||
return out[i].Out > out[j].Out
|
||||
}
|
||||
return out[i].Tag < out[j].Tag
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// CurrencyTotal is the bottom line for one currency.
|
||||
type CurrencyTotal struct {
|
||||
Currency string
|
||||
In int64
|
||||
Out int64
|
||||
Digits int
|
||||
}
|
||||
|
||||
// Net is inflow minus outflow.
|
||||
func (c CurrencyTotal) Net() int64 { return c.In - c.Out }
|
||||
|
||||
// Totals sums the per-tag rows per currency. Currencies are never combined,
|
||||
// because the tool holds no exchange rates.
|
||||
func Totals(rows []TagTotal) []CurrencyTotal {
|
||||
acc := map[string]*CurrencyTotal{}
|
||||
for _, r := range rows {
|
||||
c, ok := acc[r.Currency]
|
||||
if !ok {
|
||||
c = &CurrencyTotal{Currency: r.Currency, Digits: r.Digits}
|
||||
acc[r.Currency] = c
|
||||
}
|
||||
c.In += r.In
|
||||
c.Out += r.Out
|
||||
}
|
||||
out := make([]CurrencyTotal, 0, len(acc))
|
||||
for _, c := range acc {
|
||||
out = append(out, *c)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Currency < out[j].Currency })
|
||||
return out
|
||||
}
|
||||
|
||||
// Months lists the distinct YYYY-MM present in txns, most recent first.
|
||||
func Months(txns []model.Transaction) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, t := range txns {
|
||||
if len(t.Date) >= 7 {
|
||||
seen[t.Date[:7]] = true
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for m := range seen {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(out)))
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
// Package rules applies the ordered glob rules from rules.toml to
|
||||
// transactions, deciding their automatic tag and whether they are a transfer
|
||||
// between the user's own accounts.
|
||||
//
|
||||
// Only the rule_* columns are ever written. Manual edits made in the TUI live
|
||||
// in separate columns and survive any number of re-runs.
|
||||
package rules
|
||||
|
||||
import (
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
"git.petrovv.com/nikola/money/internal/glob"
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
"git.petrovv.com/nikola/money/internal/store"
|
||||
)
|
||||
|
||||
// Engine evaluates rules in file order; the first match wins.
|
||||
type Engine struct {
|
||||
rules []config.Rule
|
||||
}
|
||||
|
||||
// New builds an engine from the parsed rules file.
|
||||
func New(r *config.Rules) *Engine {
|
||||
return &Engine{rules: r.Rule}
|
||||
}
|
||||
|
||||
// Match returns the first rule matching a transaction on the given account, or
|
||||
// nil if none does. Every pattern a rule sets must match: a rule with both
|
||||
// match and counterparty is an "and", not an "or".
|
||||
func (e *Engine) Match(accountSlug string, t model.Transaction) *config.Rule {
|
||||
var (
|
||||
description = model.NormalizeDescription(t.Description)
|
||||
counterparty = model.NormalizeDescription(t.Counterparty)
|
||||
kind = model.NormalizeDescription(t.Type)
|
||||
)
|
||||
for i := range e.rules {
|
||||
r := &e.rules[i]
|
||||
if r.Account != "" && r.Account != accountSlug {
|
||||
continue
|
||||
}
|
||||
if r.Match != "" && !glob.Match(r.Match, description) {
|
||||
continue
|
||||
}
|
||||
if r.Counterparty != "" && !glob.Match(r.Counterparty, counterparty) {
|
||||
continue
|
||||
}
|
||||
if r.Type != "" && !glob.Match(r.Type, kind) {
|
||||
continue
|
||||
}
|
||||
return r
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ApplyTxn returns the tag and transfer flag for a transaction. An unmatched
|
||||
// transaction gets an empty tag and is not a transfer.
|
||||
func (e *Engine) ApplyTxn(accountSlug string, t model.Transaction) (tag string, transfer bool) {
|
||||
if r := e.Match(accountSlug, t); r != nil {
|
||||
return r.Tag, r.Transfer
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Apply is the description-only shorthand, for callers that have nothing else.
|
||||
func (e *Engine) Apply(accountSlug, description string) (tag string, transfer bool) {
|
||||
return e.ApplyTxn(accountSlug, model.Transaction{Description: description})
|
||||
}
|
||||
|
||||
// Retag recomputes rule verdicts for every transaction in the index and
|
||||
// writes them back. It returns how many rows changed.
|
||||
func (e *Engine) Retag(db *store.DB) (int, error) {
|
||||
txns, err := db.Transactions(store.Filter{})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var changed []store.RuleAssignment
|
||||
for _, t := range txns {
|
||||
tag, transfer := e.ApplyTxn(t.AccountSlug, t)
|
||||
if tag == t.RuleTag && transfer == t.RuleTransfer {
|
||||
continue
|
||||
}
|
||||
changed = append(changed, store.RuleAssignment{ID: t.ID, Tag: tag, Transfer: transfer})
|
||||
}
|
||||
if len(changed) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := db.ApplyRuleResults(changed); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(changed), nil
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package rules
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
"git.petrovv.com/nikola/money/internal/store"
|
||||
)
|
||||
|
||||
func testDB(t *testing.T) *store.DB {
|
||||
t.Helper()
|
||||
db, err := store.Open(filepath.Join(t.TempDir(), "index.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
// seed inserts transactions with no rule verdict yet.
|
||||
func seed(t *testing.T, db *store.DB, descriptions ...string) int64 {
|
||||
t.Helper()
|
||||
accountID, err := db.UpsertAccount(model.Account{
|
||||
Slug: "checking", Name: "Checking", Currency: "EUR", MinorDigits: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceID, err := db.SourceFile(accountID, "checking/st.csv", "sha", "2026-01-01T00:00:00Z")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, d := range descriptions {
|
||||
added, err := db.InsertTransaction(model.Transaction{
|
||||
AccountID: accountID,
|
||||
SourceFileID: sourceID,
|
||||
Fingerprint: d,
|
||||
Date: "2026-01-0" + string(rune('1'+i)),
|
||||
Description: d,
|
||||
AmountMinor: -1000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !added {
|
||||
t.Fatalf("seed %q was not inserted", d)
|
||||
}
|
||||
}
|
||||
return accountID
|
||||
}
|
||||
|
||||
func byDescription(t *testing.T, db *store.DB, desc string) model.Transaction {
|
||||
t.Helper()
|
||||
txns, err := db.Transactions(store.Filter{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, x := range txns {
|
||||
if x.Description == desc {
|
||||
return x
|
||||
}
|
||||
}
|
||||
t.Fatalf("no transaction with description %q", desc)
|
||||
return model.Transaction{}
|
||||
}
|
||||
|
||||
// A manual tag must outlive any number of rule changes and re-runs. This is
|
||||
// the property that makes retag safe to run at will.
|
||||
func TestManualTagSurvivesRetag(t *testing.T) {
|
||||
db := testDB(t)
|
||||
seed(t, db, "LIDL SOFIA", "SOME RANDOM SHOP")
|
||||
|
||||
engine := New(&config.Rules{Rule: []config.Rule{
|
||||
{Match: "*LIDL*", Tag: "groceries"},
|
||||
}})
|
||||
if _, err := engine.Retag(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The user hand-tags the row the rules did not match, and overrides the
|
||||
// one they did.
|
||||
shop := byDescription(t, db, "SOME RANDOM SHOP")
|
||||
if err := db.SetManualTag(shop.ID, "hobby"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lidl := byDescription(t, db, "LIDL SOFIA")
|
||||
if err := db.SetManualTag(lidl.ID, "party supplies"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Rules change completely, and are re-applied twice for good measure.
|
||||
engine = New(&config.Rules{Rule: []config.Rule{
|
||||
{Match: "*LIDL*", Tag: "food"},
|
||||
{Match: "*SHOP*", Tag: "shopping"},
|
||||
}})
|
||||
for range 2 {
|
||||
if _, err := engine.Retag(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
shop = byDescription(t, db, "SOME RANDOM SHOP")
|
||||
if got := shop.Tag(); got != "hobby" {
|
||||
t.Errorf("manual tag = %q, want %q (manual tags must win)", got, "hobby")
|
||||
}
|
||||
if shop.RuleTag != "shopping" {
|
||||
t.Errorf("rule_tag = %q, want %q (rules must still be recorded underneath)", shop.RuleTag, "shopping")
|
||||
}
|
||||
|
||||
lidl = byDescription(t, db, "LIDL SOFIA")
|
||||
if got := lidl.Tag(); got != "party supplies" {
|
||||
t.Errorf("manual override = %q, want %q", got, "party supplies")
|
||||
}
|
||||
|
||||
// Clearing the override falls back to whatever the rules now say.
|
||||
if err := db.ClearOverrides(lidl.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := byDescription(t, db, "LIDL SOFIA").Tag(); got != "food" {
|
||||
t.Errorf("after clearing override, tag = %q, want %q", got, "food")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstMatchWins(t *testing.T) {
|
||||
engine := New(&config.Rules{Rule: []config.Rule{
|
||||
{Match: "*LIDL EXPRESS*", Tag: "snacks"},
|
||||
{Match: "*LIDL*", Tag: "groceries"},
|
||||
}})
|
||||
if tag, _ := engine.Apply("checking", "CARD LIDL EXPRESS 12"); tag != "snacks" {
|
||||
t.Errorf("tag = %q, want snacks (earlier rule must win)", tag)
|
||||
}
|
||||
if tag, _ := engine.Apply("checking", "CARD LIDL 12"); tag != "groceries" {
|
||||
t.Errorf("tag = %q, want groceries", tag)
|
||||
}
|
||||
if tag, _ := engine.Apply("checking", "SOMETHING ELSE"); tag != "" {
|
||||
t.Errorf("tag = %q, want empty for an unmatched description", tag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountScopedRule(t *testing.T) {
|
||||
engine := New(&config.Rules{Rule: []config.Rule{
|
||||
{Match: "*TRANSFER*", Tag: "transfer", Transfer: true, Account: "savings"},
|
||||
{Match: "*TRANSFER*", Tag: "misc"},
|
||||
}})
|
||||
|
||||
tag, transfer := engine.Apply("savings", "TRANSFER FROM CHECKING")
|
||||
if tag != "transfer" || !transfer {
|
||||
t.Errorf("savings: tag=%q transfer=%v, want transfer/true", tag, transfer)
|
||||
}
|
||||
tag, transfer = engine.Apply("checking", "TRANSFER FROM CHECKING")
|
||||
if tag != "misc" || transfer {
|
||||
t.Errorf("checking: tag=%q transfer=%v, want misc/false", tag, transfer)
|
||||
}
|
||||
}
|
||||
|
||||
// Movements between the user's own accounts are often only identifiable by
|
||||
// the counterparty IBAN, so rules can match on it.
|
||||
func TestCounterpartyAndTypeRules(t *testing.T) {
|
||||
engine := New(&config.Rules{Rule: []config.Rule{
|
||||
{Counterparty: "SI56123456789012345", Tag: "transfer", Transfer: true},
|
||||
{Type: "CARD_PAYMENT", Match: "*LIDL*", Tag: "groceries"},
|
||||
{Type: "ATM", Tag: "cash"},
|
||||
}})
|
||||
|
||||
// Counterparty alone is enough, whatever the description says.
|
||||
tag, transfer := engine.ApplyTxn("checking", model.Transaction{
|
||||
Description: "Standing order", Counterparty: "SI56123456789012345",
|
||||
})
|
||||
if tag != "transfer" || !transfer {
|
||||
t.Errorf("tag=%q transfer=%v, want transfer/true", tag, transfer)
|
||||
}
|
||||
|
||||
// A rule setting several patterns requires all of them to match.
|
||||
tag, _ = engine.ApplyTxn("checking", model.Transaction{
|
||||
Description: "LIDL SOFIA", Type: "CARD_PAYMENT",
|
||||
})
|
||||
if tag != "groceries" {
|
||||
t.Errorf("tag = %q, want groceries", tag)
|
||||
}
|
||||
tag, _ = engine.ApplyTxn("checking", model.Transaction{
|
||||
Description: "LIDL SOFIA", Type: "TRANSFER",
|
||||
})
|
||||
if tag != "" {
|
||||
t.Errorf("tag = %q, want empty: the type does not match", tag)
|
||||
}
|
||||
|
||||
// Type alone works too.
|
||||
if tag, _ := engine.ApplyTxn("checking", model.Transaction{
|
||||
Description: "WITHDRAWAL", Type: "ATM",
|
||||
}); tag != "cash" {
|
||||
t.Errorf("tag = %q, want cash", tag)
|
||||
}
|
||||
}
|
||||
|
||||
// Transfers can also be overridden by hand, independently of the tag.
|
||||
func TestManualTransferOverride(t *testing.T) {
|
||||
db := testDB(t)
|
||||
seed(t, db, "TRANSFER TO SAVINGS")
|
||||
|
||||
engine := New(&config.Rules{Rule: []config.Rule{
|
||||
{Match: "*TO SAVINGS*", Tag: "transfer", Transfer: true},
|
||||
}})
|
||||
if _, err := engine.Retag(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
txn := byDescription(t, db, "TRANSFER TO SAVINGS")
|
||||
if !txn.IsTransfer() {
|
||||
t.Fatal("expected the rule to mark this as a transfer")
|
||||
}
|
||||
|
||||
no := false
|
||||
if err := db.SetManualTransfer(txn.ID, &no); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := engine.Retag(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if byDescription(t, db, "TRANSFER TO SAVINGS").IsTransfer() {
|
||||
t.Error("manual transfer override was lost on retag")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUntaggedFilter(t *testing.T) {
|
||||
db := testDB(t)
|
||||
seed(t, db, "LIDL SOFIA", "SOME RANDOM SHOP")
|
||||
|
||||
engine := New(&config.Rules{Rule: []config.Rule{{Match: "*LIDL*", Tag: "groceries"}}})
|
||||
if _, err := engine.Retag(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
untagged, err := db.Transactions(store.Filter{Untagged: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(untagged) != 1 || untagged[0].Description != "SOME RANDOM SHOP" {
|
||||
t.Fatalf("untagged = %+v, want only SOME RANDOM SHOP", untagged)
|
||||
}
|
||||
|
||||
// A manual tag must remove the row from the untagged view.
|
||||
if err := db.SetManualTag(untagged[0].ID, "hobby"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
untagged, err = db.Transactions(store.Filter{Untagged: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(untagged) != 0 {
|
||||
t.Errorf("untagged = %+v, want none", untagged)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
// Package store is the SQLite index over the statements. It is entirely
|
||||
// rebuildable: delete .money/index.db and re-import to get it back, except for
|
||||
// manual tags and manual transfer overrides, which live only here.
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
)
|
||||
|
||||
// DB wraps the SQLite handle.
|
||||
type DB struct {
|
||||
sql *sql.DB
|
||||
}
|
||||
|
||||
const schema = `
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS accounts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
currency TEXT NOT NULL,
|
||||
minor_digits INTEGER NOT NULL DEFAULT 2
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_files (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
account_id INTEGER NOT NULL REFERENCES accounts(id),
|
||||
path TEXT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
imported_at TEXT NOT NULL,
|
||||
UNIQUE(account_id, path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS transactions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
account_id INTEGER NOT NULL REFERENCES accounts(id),
|
||||
source_file_id INTEGER NOT NULL REFERENCES source_files(id),
|
||||
fingerprint TEXT NOT NULL,
|
||||
date TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
amount_minor INTEGER NOT NULL,
|
||||
counterparty TEXT NOT NULL DEFAULT '',
|
||||
type TEXT NOT NULL DEFAULT '',
|
||||
balance_minor INTEGER,
|
||||
rule_tag TEXT,
|
||||
manual_tag TEXT,
|
||||
rule_transfer INTEGER NOT NULL DEFAULT 0,
|
||||
manual_transfer INTEGER,
|
||||
UNIQUE(account_id, fingerprint)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_txn_date ON transactions(date);
|
||||
CREATE INDEX IF NOT EXISTS idx_txn_account ON transactions(account_id);
|
||||
`
|
||||
|
||||
// migrations bring an index created by an older build up to date. SQLite
|
||||
// errors on a duplicate column, which is how we detect "already applied".
|
||||
var migrations = []string{
|
||||
`ALTER TABLE transactions ADD COLUMN counterparty TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE transactions ADD COLUMN type TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE transactions ADD COLUMN balance_minor INTEGER`,
|
||||
}
|
||||
|
||||
// migrate applies any column that this index is missing.
|
||||
func migrate(db *sql.DB) error {
|
||||
have, err := columns(db, "transactions")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, stmt := range migrations {
|
||||
name := addedColumn(stmt)
|
||||
if have[name] {
|
||||
continue
|
||||
}
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
return fmt.Errorf("migrate (%s): %w", stmt, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func columns(db *sql.DB, table string) (map[string]bool, error) {
|
||||
rows, err := db.Query(`SELECT name FROM pragma_table_info(?)`, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect %s: %w", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// addedColumn pulls the column name out of an ADD COLUMN statement.
|
||||
func addedColumn(stmt string) string {
|
||||
_, rest, ok := strings.Cut(stmt, "ADD COLUMN ")
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
name, _, _ := strings.Cut(rest, " ")
|
||||
return name
|
||||
}
|
||||
|
||||
// Open opens (creating if needed) the index at path.
|
||||
func Open(path string) (*DB, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create state dir: %w", err)
|
||||
}
|
||||
sqlDB, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open index %s: %w", path, err)
|
||||
}
|
||||
if _, err := sqlDB.Exec(schema); err != nil {
|
||||
sqlDB.Close()
|
||||
return nil, fmt.Errorf("apply schema: %w", err)
|
||||
}
|
||||
if err := migrate(sqlDB); err != nil {
|
||||
sqlDB.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &DB{sql: sqlDB}, nil
|
||||
}
|
||||
|
||||
// Close releases the underlying handle.
|
||||
func (d *DB) Close() error { return d.sql.Close() }
|
||||
|
||||
// UpsertAccount inserts or updates an account by slug and returns its id.
|
||||
func (d *DB) UpsertAccount(a model.Account) (int64, error) {
|
||||
_, err := d.sql.Exec(`
|
||||
INSERT INTO accounts (slug, name, currency, minor_digits)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(slug) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
currency = excluded.currency,
|
||||
minor_digits = excluded.minor_digits`,
|
||||
a.Slug, a.Name, a.Currency, a.MinorDigits)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("upsert account %s: %w", a.Slug, err)
|
||||
}
|
||||
var id int64
|
||||
if err := d.sql.QueryRow(`SELECT id FROM accounts WHERE slug = ?`, a.Slug).Scan(&id); err != nil {
|
||||
return 0, fmt.Errorf("read account id %s: %w", a.Slug, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Accounts lists every known account, ordered by slug.
|
||||
func (d *DB) Accounts() ([]model.Account, error) {
|
||||
rows, err := d.sql.Query(`SELECT id, slug, name, currency, minor_digits FROM accounts ORDER BY slug`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list accounts: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []model.Account
|
||||
for rows.Next() {
|
||||
var a model.Account
|
||||
if err := rows.Scan(&a.ID, &a.Slug, &a.Name, &a.Currency, &a.MinorDigits); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SourceFile records that a statement file was imported, returning its id.
|
||||
func (d *DB) SourceFile(accountID int64, path, sha, importedAt string) (int64, error) {
|
||||
_, err := d.sql.Exec(`
|
||||
INSERT INTO source_files (account_id, path, sha256, imported_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(account_id, path) DO UPDATE SET
|
||||
sha256 = excluded.sha256,
|
||||
imported_at = excluded.imported_at`,
|
||||
accountID, path, sha, importedAt)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("record source file %s: %w", path, err)
|
||||
}
|
||||
var id int64
|
||||
if err := d.sql.QueryRow(
|
||||
`SELECT id FROM source_files WHERE account_id = ? AND path = ?`, accountID, path).Scan(&id); err != nil {
|
||||
return 0, fmt.Errorf("read source file id %s: %w", path, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// SourceFileSHA returns the recorded checksum for a statement file, and whether
|
||||
// it has been imported before.
|
||||
func (d *DB) SourceFileSHA(accountID int64, path string) (string, bool, error) {
|
||||
var sha string
|
||||
err := d.sql.QueryRow(
|
||||
`SELECT sha256 FROM source_files WHERE account_id = ? AND path = ?`, accountID, path).Scan(&sha)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return sha, true, nil
|
||||
}
|
||||
|
||||
// InsertTransaction adds a transaction unless its fingerprint already exists
|
||||
// for that account. It reports whether a new row was created.
|
||||
//
|
||||
// Existing rows are deliberately left untouched: re-importing an overlapping
|
||||
// statement must never clobber tags the user set by hand.
|
||||
func (d *DB) InsertTransaction(t model.Transaction) (bool, error) {
|
||||
var balance any
|
||||
if t.BalanceMinor != nil {
|
||||
balance = *t.BalanceMinor
|
||||
}
|
||||
res, err := d.sql.Exec(`
|
||||
INSERT INTO transactions
|
||||
(account_id, source_file_id, fingerprint, date, description, amount_minor,
|
||||
counterparty, type, balance_minor, rule_tag, rule_transfer)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULLIF(?, ''), ?)
|
||||
ON CONFLICT(account_id, fingerprint) DO NOTHING`,
|
||||
t.AccountID, t.SourceFileID, t.Fingerprint, t.Date, t.Description,
|
||||
t.AmountMinor, t.Counterparty, t.Type, balance, t.RuleTag, boolToInt(t.RuleTransfer))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("insert transaction: %w", err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// Filter narrows a transaction query.
|
||||
type Filter struct {
|
||||
AccountSlug string
|
||||
Untagged bool // only rows with no effective tag
|
||||
Month string // YYYY-MM
|
||||
Search string // case-insensitive substring of the description
|
||||
Limit int
|
||||
}
|
||||
|
||||
// Transactions returns rows matching f, newest first.
|
||||
func (d *DB) Transactions(f Filter) ([]model.Transaction, error) {
|
||||
q := `
|
||||
SELECT t.id, t.account_id, a.slug, a.currency, a.minor_digits,
|
||||
t.fingerprint, t.date, t.description, t.amount_minor,
|
||||
COALESCE(s.path, ''), t.counterparty, t.type, t.balance_minor,
|
||||
COALESCE(t.rule_tag, ''), COALESCE(t.manual_tag, ''),
|
||||
t.rule_transfer, t.manual_transfer
|
||||
FROM transactions t
|
||||
JOIN accounts a ON a.id = t.account_id
|
||||
LEFT JOIN source_files s ON s.id = t.source_file_id
|
||||
WHERE 1 = 1`
|
||||
var args []any
|
||||
if f.AccountSlug != "" {
|
||||
q += ` AND a.slug = ?`
|
||||
args = append(args, f.AccountSlug)
|
||||
}
|
||||
if f.Untagged {
|
||||
q += ` AND COALESCE(NULLIF(t.manual_tag, ''), NULLIF(t.rule_tag, '')) IS NULL`
|
||||
}
|
||||
if f.Month != "" {
|
||||
q += ` AND substr(t.date, 1, 7) = ?`
|
||||
args = append(args, f.Month)
|
||||
}
|
||||
q += ` ORDER BY t.date DESC, t.id DESC`
|
||||
// Search and Limit are applied in Go: SQLite's upper()/LIKE fold ASCII
|
||||
// only, which would silently fail on Cyrillic statement descriptions.
|
||||
|
||||
rows, err := d.sql.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query transactions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
needle := model.NormalizeDescription(f.Search)
|
||||
var out []model.Transaction
|
||||
for rows.Next() {
|
||||
var (
|
||||
t model.Transaction
|
||||
ruleTransfer int
|
||||
manualTransfer sql.NullInt64
|
||||
balance sql.NullInt64
|
||||
)
|
||||
if err := rows.Scan(&t.ID, &t.AccountID, &t.AccountSlug, &t.Currency, &t.MinorDigits,
|
||||
&t.Fingerprint, &t.Date, &t.Description, &t.AmountMinor, &t.SourcePath,
|
||||
&t.Counterparty, &t.Type, &balance,
|
||||
&t.RuleTag, &t.ManualTag, &ruleTransfer, &manualTransfer); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.RuleTransfer = ruleTransfer != 0
|
||||
if balance.Valid {
|
||||
v := balance.Int64
|
||||
t.BalanceMinor = &v
|
||||
}
|
||||
if manualTransfer.Valid {
|
||||
v := manualTransfer.Int64 != 0
|
||||
t.ManualTransfer = &v
|
||||
}
|
||||
if needle != "" && !strings.Contains(model.NormalizeDescription(t.Description), needle) {
|
||||
continue
|
||||
}
|
||||
out = append(out, t)
|
||||
if f.Limit > 0 && len(out) >= f.Limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetManualTag sets (or, with an empty tag, clears) the user's tag override.
|
||||
func (d *DB) SetManualTag(id int64, tag string) error {
|
||||
_, err := d.sql.Exec(`UPDATE transactions SET manual_tag = NULLIF(?, '') WHERE id = ?`, tag, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// SetManualTransfer sets the user's transfer override; nil restores the
|
||||
// rule-derived value.
|
||||
func (d *DB) SetManualTransfer(id int64, v *bool) error {
|
||||
var arg any
|
||||
if v != nil {
|
||||
arg = boolToInt(*v)
|
||||
}
|
||||
_, err := d.sql.Exec(`UPDATE transactions SET manual_transfer = ? WHERE id = ?`, arg, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// ClearOverrides drops both manual overrides for a transaction.
|
||||
func (d *DB) ClearOverrides(id int64) error {
|
||||
_, err := d.sql.Exec(
|
||||
`UPDATE transactions SET manual_tag = NULL, manual_transfer = NULL WHERE id = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// RuleAssignment is one row's recomputed rule verdict.
|
||||
type RuleAssignment struct {
|
||||
ID int64
|
||||
Tag string
|
||||
Transfer bool
|
||||
}
|
||||
|
||||
// ApplyRuleResults rewrites rule_tag and rule_transfer for every listed row in
|
||||
// a single transaction. Manual columns are never touched.
|
||||
func (d *DB) ApplyRuleResults(rs []RuleAssignment) error {
|
||||
tx, err := d.sql.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(
|
||||
`UPDATE transactions SET rule_tag = NULLIF(?, ''), rule_transfer = ? WHERE id = ?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for _, r := range rs {
|
||||
if _, err := stmt.Exec(r.Tag, boolToInt(r.Transfer), r.ID); err != nil {
|
||||
return fmt.Errorf("apply rules to txn %d: %w", r.ID, err)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// Balance sums every transaction in an account, transfers included: a transfer
|
||||
// really does move the account's money.
|
||||
func (d *DB) Balance(accountID int64) (int64, error) {
|
||||
var v sql.NullInt64
|
||||
err := d.sql.QueryRow(
|
||||
`SELECT SUM(amount_minor) FROM transactions WHERE account_id = ?`, accountID).Scan(&v)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return v.Int64, nil
|
||||
}
|
||||
|
||||
// Count returns the number of transactions in an account.
|
||||
func (d *DB) Count(accountID int64) (int, error) {
|
||||
var n int
|
||||
err := d.sql.QueryRow(
|
||||
`SELECT COUNT(*) FROM transactions WHERE account_id = ?`, accountID).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Tags lists every effective tag in use, for completion in the TUI.
|
||||
func (d *DB) Tags() ([]string, error) {
|
||||
rows, err := d.sql.Query(`
|
||||
SELECT DISTINCT COALESCE(NULLIF(manual_tag, ''), NULLIF(rule_tag, '')) AS tag
|
||||
FROM transactions WHERE tag IS NOT NULL ORDER BY tag`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var s string
|
||||
if err := rows.Scan(&s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func boolToInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
// Package tui is the terminal interface: browse accounts, tag transactions,
|
||||
// mark transfers by hand, and see where the money went.
|
||||
package tui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/charmbracelet/bubbles/table"
|
||||
"github.com/charmbracelet/bubbles/textinput"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
"git.petrovv.com/nikola/money/internal/importer"
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
"git.petrovv.com/nikola/money/internal/report"
|
||||
"git.petrovv.com/nikola/money/internal/rules"
|
||||
"git.petrovv.com/nikola/money/internal/store"
|
||||
)
|
||||
|
||||
type view int
|
||||
|
||||
const (
|
||||
viewAccounts view = iota
|
||||
viewTxns
|
||||
viewReport
|
||||
)
|
||||
|
||||
// input is the modal state: the transaction list is read-only until the user
|
||||
// starts typing a tag or a search.
|
||||
type input int
|
||||
|
||||
const (
|
||||
inputNone input = iota
|
||||
inputTag
|
||||
inputSearch
|
||||
)
|
||||
|
||||
// Model is the root Bubble Tea model.
|
||||
type Model struct {
|
||||
root string
|
||||
db *store.DB
|
||||
accounts []*config.Account
|
||||
engine *rules.Engine
|
||||
|
||||
view view
|
||||
input input
|
||||
|
||||
accountTable table.Model
|
||||
txnTable table.Model
|
||||
reportTable table.Model
|
||||
text textinput.Model
|
||||
|
||||
txns []model.Transaction // rows currently shown in txnTable
|
||||
|
||||
filter store.Filter
|
||||
onlyUntagged bool
|
||||
|
||||
status string
|
||||
err error
|
||||
width int
|
||||
height int
|
||||
}
|
||||
|
||||
var (
|
||||
titleStyle = lipgloss.NewStyle().Bold(true).Padding(0, 1)
|
||||
statusStyle = lipgloss.NewStyle().Faint(true).Padding(0, 1)
|
||||
errStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("9")).Padding(0, 1)
|
||||
helpStyle = lipgloss.NewStyle().Faint(true).Padding(0, 1)
|
||||
)
|
||||
|
||||
// Run starts the interface.
|
||||
func Run(root string, db *store.DB, accounts []*config.Account, engine *rules.Engine) error {
|
||||
m := New(root, db, accounts, engine)
|
||||
if err := m.reload(); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tea.NewProgram(m, tea.WithAltScreen()).Run()
|
||||
return err
|
||||
}
|
||||
|
||||
// New builds the root model.
|
||||
func New(root string, db *store.DB, accounts []*config.Account, engine *rules.Engine) *Model {
|
||||
ti := textinput.New()
|
||||
ti.Prompt = ""
|
||||
ti.CharLimit = 64
|
||||
|
||||
styles := table.DefaultStyles()
|
||||
styles.Header = styles.Header.Bold(true)
|
||||
styles.Selected = styles.Selected.Bold(true).Foreground(lipgloss.Color("15")).Background(lipgloss.Color("62"))
|
||||
|
||||
newTable := func(cols []table.Column) table.Model {
|
||||
return table.New(table.WithColumns(cols), table.WithFocused(true),
|
||||
table.WithHeight(15), table.WithStyles(styles))
|
||||
}
|
||||
|
||||
return &Model{
|
||||
root: root,
|
||||
db: db,
|
||||
accounts: accounts,
|
||||
engine: engine,
|
||||
view: viewAccounts,
|
||||
text: ti,
|
||||
accountTable: newTable([]table.Column{
|
||||
{Title: "Account", Width: 20},
|
||||
{Title: "Balance", Width: 14},
|
||||
{Title: "Cur", Width: 4},
|
||||
{Title: "Txns", Width: 6},
|
||||
}),
|
||||
txnTable: newTable([]table.Column{
|
||||
{Title: "Date", Width: 10},
|
||||
{Title: "Account", Width: 12},
|
||||
{Title: "Amount", Width: 12},
|
||||
{Title: "Tag", Width: 14},
|
||||
{Title: "T", Width: 1},
|
||||
{Title: "Description", Width: 40},
|
||||
}),
|
||||
reportTable: newTable([]table.Column{
|
||||
{Title: "Tag", Width: 20},
|
||||
{Title: "Cur", Width: 4},
|
||||
{Title: "Out", Width: 14},
|
||||
{Title: "In", Width: 14},
|
||||
{Title: "Net", Width: 14},
|
||||
{Title: "N", Width: 5},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// Init implements tea.Model.
|
||||
func (m *Model) Init() tea.Cmd { return nil }
|
||||
|
||||
// reload refreshes whichever view is active from the index.
|
||||
func (m *Model) reload() error {
|
||||
if err := m.reloadAccounts(); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.reloadTxns()
|
||||
}
|
||||
|
||||
func (m *Model) reloadAccounts() error {
|
||||
accounts, err := m.db.Accounts()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows := make([]table.Row, 0, len(accounts))
|
||||
for _, a := range accounts {
|
||||
bal, err := m.db.Balance(a.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
n, err := m.db.Count(a.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows = append(rows, table.Row{
|
||||
a.Name, model.FormatMinor(bal, a.MinorDigits), a.Currency, fmt.Sprintf("%d", n),
|
||||
})
|
||||
}
|
||||
m.accountTable.SetRows(rows)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Model) reloadTxns() error {
|
||||
f := m.filter
|
||||
f.Untagged = m.onlyUntagged
|
||||
txns, err := m.db.Transactions(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.txns = txns
|
||||
|
||||
rows := make([]table.Row, 0, len(txns))
|
||||
for _, t := range txns {
|
||||
transfer := ""
|
||||
if t.IsTransfer() {
|
||||
transfer = "T"
|
||||
}
|
||||
tag := t.Tag()
|
||||
if tag != "" && t.ManualTag != "" {
|
||||
tag += "*" // marks a manual override
|
||||
}
|
||||
rows = append(rows, table.Row{
|
||||
t.Date, t.AccountSlug, t.FormatAmount(), tag, transfer, t.Description,
|
||||
})
|
||||
}
|
||||
// Keep the cursor in range after the row count shrinks (e.g. tagging the
|
||||
// last untagged row while the untagged filter is on).
|
||||
cursor := m.txnTable.Cursor()
|
||||
m.txnTable.SetRows(rows)
|
||||
if cursor >= len(rows) {
|
||||
cursor = len(rows) - 1
|
||||
}
|
||||
if cursor < 0 {
|
||||
cursor = 0
|
||||
}
|
||||
m.txnTable.SetCursor(cursor)
|
||||
|
||||
m.reloadReport(txns)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Model) reloadReport(txns []model.Transaction) {
|
||||
rows := report.ByTag(txns)
|
||||
out := make([]table.Row, 0, len(rows)+2)
|
||||
for _, r := range rows {
|
||||
out = append(out, table.Row{
|
||||
r.Tag, r.Currency,
|
||||
model.FormatMinor(r.Out, r.Digits),
|
||||
model.FormatMinor(r.In, r.Digits),
|
||||
model.FormatMinor(r.Net(), r.Digits),
|
||||
fmt.Sprintf("%d", r.Count),
|
||||
})
|
||||
}
|
||||
for _, c := range report.Totals(rows) {
|
||||
out = append(out, table.Row{
|
||||
"TOTAL", c.Currency,
|
||||
model.FormatMinor(c.Out, c.Digits),
|
||||
model.FormatMinor(c.In, c.Digits),
|
||||
model.FormatMinor(c.Net(), c.Digits),
|
||||
"",
|
||||
})
|
||||
}
|
||||
m.reportTable.SetRows(out)
|
||||
}
|
||||
|
||||
// selected returns the transaction under the cursor, if any.
|
||||
func (m *Model) selected() (model.Transaction, bool) {
|
||||
i := m.txnTable.Cursor()
|
||||
if i < 0 || i >= len(m.txns) {
|
||||
return model.Transaction{}, false
|
||||
}
|
||||
return m.txns[i], true
|
||||
}
|
||||
|
||||
// Update implements tea.Model.
|
||||
func (m *Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.WindowSizeMsg:
|
||||
m.width, m.height = msg.Width, msg.Height
|
||||
m.resize()
|
||||
return m, nil
|
||||
|
||||
case tea.KeyMsg:
|
||||
if m.input != inputNone {
|
||||
return m.updateInput(msg)
|
||||
}
|
||||
return m.updateNormal(msg)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *Model) resize() {
|
||||
h := m.height - 6 // title, status, help, padding
|
||||
if h < 3 {
|
||||
h = 3
|
||||
}
|
||||
m.accountTable.SetHeight(h)
|
||||
m.txnTable.SetHeight(h)
|
||||
m.reportTable.SetHeight(h)
|
||||
|
||||
// Give the description column whatever is left over.
|
||||
if m.width > 0 {
|
||||
cols := m.txnTable.Columns()
|
||||
fixed := 0
|
||||
for _, c := range cols[:len(cols)-1] {
|
||||
fixed += c.Width + 2
|
||||
}
|
||||
desc := m.width - fixed - 4
|
||||
if desc < 20 {
|
||||
desc = 20
|
||||
}
|
||||
cols[len(cols)-1].Width = desc
|
||||
m.txnTable.SetColumns(cols)
|
||||
}
|
||||
}
|
||||
|
||||
// updateInput handles typing into the tag or search prompt.
|
||||
func (m *Model) updateInput(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.Type {
|
||||
case tea.KeyEsc:
|
||||
m.input = inputNone
|
||||
m.text.Blur()
|
||||
m.text.SetValue("")
|
||||
return m, nil
|
||||
case tea.KeyEnter:
|
||||
value := strings.TrimSpace(m.text.Value())
|
||||
mode := m.input
|
||||
m.input = inputNone
|
||||
m.text.Blur()
|
||||
m.text.SetValue("")
|
||||
|
||||
switch mode {
|
||||
case inputTag:
|
||||
t, ok := m.selected()
|
||||
if !ok {
|
||||
return m, nil
|
||||
}
|
||||
if err := m.db.SetManualTag(t.ID, value); err != nil {
|
||||
m.err = err
|
||||
return m, nil
|
||||
}
|
||||
if value == "" {
|
||||
m.status = "cleared manual tag"
|
||||
} else {
|
||||
m.status = "tagged as " + value
|
||||
}
|
||||
case inputSearch:
|
||||
m.filter.Search = value
|
||||
m.status = "search: " + value
|
||||
if value == "" {
|
||||
m.status = "search cleared"
|
||||
}
|
||||
}
|
||||
m.err = m.reloadTxns()
|
||||
return m, nil
|
||||
}
|
||||
|
||||
var cmd tea.Cmd
|
||||
m.text, cmd = m.text.Update(msg)
|
||||
return m, cmd
|
||||
}
|
||||
|
||||
// updateNormal handles the main keymap.
|
||||
func (m *Model) updateNormal(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch msg.String() {
|
||||
case "q", "ctrl+c":
|
||||
return m, tea.Quit
|
||||
|
||||
case "1":
|
||||
m.view = viewAccounts
|
||||
return m, nil
|
||||
case "2":
|
||||
m.view = viewTxns
|
||||
return m, nil
|
||||
case "3":
|
||||
m.view = viewReport
|
||||
return m, nil
|
||||
case "tab":
|
||||
m.view = (m.view + 1) % 3
|
||||
return m, nil
|
||||
|
||||
case "enter":
|
||||
// From the account list, drill into that account's transactions.
|
||||
if m.view == viewAccounts {
|
||||
accounts, err := m.db.Accounts()
|
||||
if err != nil {
|
||||
m.err = err
|
||||
return m, nil
|
||||
}
|
||||
i := m.accountTable.Cursor()
|
||||
if i >= 0 && i < len(accounts) {
|
||||
m.filter.AccountSlug = accounts[i].Slug
|
||||
m.status = "account: " + accounts[i].Slug
|
||||
m.view = viewTxns
|
||||
m.err = m.reloadTxns()
|
||||
}
|
||||
}
|
||||
return m, nil
|
||||
|
||||
case "a":
|
||||
m.filter.AccountSlug = ""
|
||||
m.status = "all accounts"
|
||||
m.err = m.reloadTxns()
|
||||
return m, nil
|
||||
|
||||
case "u":
|
||||
m.onlyUntagged = !m.onlyUntagged
|
||||
if m.onlyUntagged {
|
||||
m.status = "showing untagged only"
|
||||
m.view = viewTxns
|
||||
} else {
|
||||
m.status = "showing all transactions"
|
||||
}
|
||||
m.err = m.reloadTxns()
|
||||
return m, nil
|
||||
|
||||
case "t":
|
||||
if m.view != viewTxns {
|
||||
return m, nil
|
||||
}
|
||||
t, ok := m.selected()
|
||||
if !ok {
|
||||
return m, nil
|
||||
}
|
||||
m.input = inputTag
|
||||
m.text.SetValue(t.ManualTag)
|
||||
m.text.CursorEnd()
|
||||
m.text.Focus()
|
||||
return m, nil
|
||||
|
||||
case "/":
|
||||
m.view = viewTxns
|
||||
m.input = inputSearch
|
||||
m.text.SetValue(m.filter.Search)
|
||||
m.text.CursorEnd()
|
||||
m.text.Focus()
|
||||
return m, nil
|
||||
|
||||
case "x":
|
||||
// Toggle the manual transfer override away from whatever is in effect.
|
||||
if m.view != viewTxns {
|
||||
return m, nil
|
||||
}
|
||||
t, ok := m.selected()
|
||||
if !ok {
|
||||
return m, nil
|
||||
}
|
||||
v := !t.IsTransfer()
|
||||
if err := m.db.SetManualTransfer(t.ID, &v); err != nil {
|
||||
m.err = err
|
||||
return m, nil
|
||||
}
|
||||
if v {
|
||||
m.status = "marked as transfer"
|
||||
} else {
|
||||
m.status = "unmarked as transfer"
|
||||
}
|
||||
m.err = m.reloadTxns()
|
||||
return m, nil
|
||||
|
||||
case "c":
|
||||
// Drop manual overrides and fall back to what the rules say.
|
||||
if m.view != viewTxns {
|
||||
return m, nil
|
||||
}
|
||||
t, ok := m.selected()
|
||||
if !ok {
|
||||
return m, nil
|
||||
}
|
||||
if err := m.db.ClearOverrides(t.ID); err != nil {
|
||||
m.err = err
|
||||
return m, nil
|
||||
}
|
||||
m.status = "manual overrides cleared"
|
||||
m.err = m.reloadTxns()
|
||||
return m, nil
|
||||
|
||||
case "r":
|
||||
n, err := m.engine.Retag(m.db)
|
||||
if err != nil {
|
||||
m.err = err
|
||||
return m, nil
|
||||
}
|
||||
m.status = fmt.Sprintf("rules re-applied, %d rows changed", n)
|
||||
m.err = m.reload()
|
||||
return m, nil
|
||||
|
||||
case "i":
|
||||
res, err := importer.Run(m.root, m.db, m.accounts, m.engine, importer.Options{})
|
||||
if err != nil {
|
||||
m.err = err
|
||||
return m, nil
|
||||
}
|
||||
_, added, skipped := res.Total()
|
||||
m.status = fmt.Sprintf("imported: %d new, %d duplicate", added, skipped)
|
||||
if failures := res.Errs(); len(failures) > 0 {
|
||||
m.err = fmt.Errorf("%s: %w", failures[0].Path, failures[0].Err)
|
||||
}
|
||||
m.err = m.reload()
|
||||
return m, nil
|
||||
}
|
||||
|
||||
var cmd tea.Cmd
|
||||
switch m.view {
|
||||
case viewAccounts:
|
||||
m.accountTable, cmd = m.accountTable.Update(msg)
|
||||
case viewTxns:
|
||||
m.txnTable, cmd = m.txnTable.Update(msg)
|
||||
case viewReport:
|
||||
m.reportTable, cmd = m.reportTable.Update(msg)
|
||||
}
|
||||
return m, cmd
|
||||
}
|
||||
|
||||
// View implements tea.Model.
|
||||
func (m *Model) View() string {
|
||||
var b strings.Builder
|
||||
|
||||
b.WriteString(titleStyle.Render(m.title()) + "\n")
|
||||
|
||||
switch m.view {
|
||||
case viewAccounts:
|
||||
b.WriteString(m.accountTable.View())
|
||||
case viewTxns:
|
||||
b.WriteString(m.txnTable.View())
|
||||
case viewReport:
|
||||
b.WriteString(m.reportTable.View())
|
||||
}
|
||||
b.WriteString("\n")
|
||||
|
||||
switch {
|
||||
case m.input == inputTag:
|
||||
b.WriteString(statusStyle.Render("tag: ") + m.text.View())
|
||||
case m.input == inputSearch:
|
||||
b.WriteString(statusStyle.Render("search: ") + m.text.View())
|
||||
case m.err != nil:
|
||||
b.WriteString(errStyle.Render("error: " + m.err.Error()))
|
||||
default:
|
||||
b.WriteString(statusStyle.Render(m.status))
|
||||
}
|
||||
b.WriteString("\n")
|
||||
b.WriteString(helpStyle.Render(m.help()))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func (m *Model) title() string {
|
||||
scope := "all accounts"
|
||||
if m.filter.AccountSlug != "" {
|
||||
scope = m.filter.AccountSlug
|
||||
}
|
||||
if m.onlyUntagged {
|
||||
scope += " · untagged"
|
||||
}
|
||||
if m.filter.Search != "" {
|
||||
scope += " · /" + m.filter.Search
|
||||
}
|
||||
|
||||
switch m.view {
|
||||
case viewAccounts:
|
||||
return "money · accounts"
|
||||
case viewReport:
|
||||
return "money · report · " + scope + " · transfers excluded"
|
||||
default:
|
||||
return fmt.Sprintf("money · transactions · %s · %d rows", scope, len(m.txns))
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Model) help() string {
|
||||
if m.input != inputNone {
|
||||
return "enter confirm · esc cancel"
|
||||
}
|
||||
switch m.view {
|
||||
case viewAccounts:
|
||||
return "enter open · 2 transactions · 3 report · i import · r retag · q quit"
|
||||
case viewReport:
|
||||
return "1 accounts · 2 transactions · u untagged · a all accounts · q quit"
|
||||
default:
|
||||
return "t tag · x transfer · c clear · / search · u untagged · a all · i import · r retag · 1 accounts · 3 report · q quit"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package tui
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/config"
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
"git.petrovv.com/nikola/money/internal/rules"
|
||||
"git.petrovv.com/nikola/money/internal/store"
|
||||
)
|
||||
|
||||
// newTestModel builds a model over an index holding two transactions, one of
|
||||
// which the rules tag as groceries.
|
||||
func newTestModel(t *testing.T) (*Model, *store.DB) {
|
||||
t.Helper()
|
||||
|
||||
db, err := store.Open(filepath.Join(t.TempDir(), "index.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
|
||||
accountID, err := db.UpsertAccount(model.Account{
|
||||
Slug: "checking", Name: "Checking", Currency: "EUR", MinorDigits: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceID, err := db.SourceFile(accountID, "checking/st.csv", "sha", "2026-01-01T00:00:00Z")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, d := range []string{"LIDL SOFIA", "SOME RANDOM SHOP"} {
|
||||
if _, err := db.InsertTransaction(model.Transaction{
|
||||
AccountID: accountID,
|
||||
SourceFileID: sourceID,
|
||||
Fingerprint: d,
|
||||
Date: "2026-01-0" + string(rune('1'+i)),
|
||||
Description: d,
|
||||
AmountMinor: -1000,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
engine := rules.New(&config.Rules{Rule: []config.Rule{{Match: "*LIDL*", Tag: "groceries"}}})
|
||||
if _, err := engine.Retag(db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
m := New(t.TempDir(), db, nil, engine)
|
||||
if err := m.reload(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Update(tea.WindowSizeMsg{Width: 120, Height: 30})
|
||||
return m, db
|
||||
}
|
||||
|
||||
// key sends a single keypress to the model.
|
||||
func key(t *testing.T, m *Model, s string) {
|
||||
t.Helper()
|
||||
var msg tea.KeyMsg
|
||||
switch s {
|
||||
case "enter":
|
||||
msg = tea.KeyMsg{Type: tea.KeyEnter}
|
||||
case "esc":
|
||||
msg = tea.KeyMsg{Type: tea.KeyEsc}
|
||||
default:
|
||||
msg = tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(s)}
|
||||
}
|
||||
m.Update(msg)
|
||||
if m.err != nil {
|
||||
t.Fatalf("key %q produced an error: %v", s, m.err)
|
||||
}
|
||||
}
|
||||
|
||||
// typeText sends each character as its own keypress, as a terminal would.
|
||||
func typeText(t *testing.T, m *Model, s string) {
|
||||
t.Helper()
|
||||
for _, r := range s {
|
||||
m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{r}})
|
||||
}
|
||||
}
|
||||
|
||||
// cursorTo moves the transaction cursor onto the row with the given description.
|
||||
func cursorTo(t *testing.T, m *Model, desc string) {
|
||||
t.Helper()
|
||||
for i, txn := range m.txns {
|
||||
if txn.Description == desc {
|
||||
m.txnTable.SetCursor(i)
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("no visible transaction with description %q", desc)
|
||||
}
|
||||
|
||||
func TestTagFlow(t *testing.T) {
|
||||
m, db := newTestModel(t)
|
||||
|
||||
key(t, m, "2") // transactions view
|
||||
cursorTo(t, m, "SOME RANDOM SHOP")
|
||||
|
||||
key(t, m, "t") // open the tag prompt
|
||||
if m.input != inputTag {
|
||||
t.Fatal("expected the tag prompt to open")
|
||||
}
|
||||
typeText(t, m, "hobby")
|
||||
key(t, m, "enter")
|
||||
|
||||
if m.input != inputNone {
|
||||
t.Error("expected the prompt to close after enter")
|
||||
}
|
||||
|
||||
txns, err := db.Transactions(store.Filter{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, txn := range txns {
|
||||
if txn.Description == "SOME RANDOM SHOP" {
|
||||
found = true
|
||||
if txn.ManualTag != "hobby" {
|
||||
t.Errorf("manual_tag = %q, want %q", txn.ManualTag, "hobby")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("transaction disappeared after tagging")
|
||||
}
|
||||
|
||||
// The tag column marks manual overrides with a trailing asterisk.
|
||||
if !strings.Contains(m.View(), "hobby*") {
|
||||
t.Error("expected the tagged row to render with a manual-override marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagPromptCancel(t *testing.T) {
|
||||
m, db := newTestModel(t)
|
||||
key(t, m, "2")
|
||||
cursorTo(t, m, "SOME RANDOM SHOP")
|
||||
|
||||
key(t, m, "t")
|
||||
typeText(t, m, "nope")
|
||||
key(t, m, "esc")
|
||||
|
||||
if m.input != inputNone {
|
||||
t.Error("expected esc to close the prompt")
|
||||
}
|
||||
txns, _ := db.Transactions(store.Filter{Untagged: true})
|
||||
if len(txns) != 1 {
|
||||
t.Errorf("esc should not have tagged anything; untagged count = %d, want 1", len(txns))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransferToggleAndClear(t *testing.T) {
|
||||
m, db := newTestModel(t)
|
||||
key(t, m, "2")
|
||||
cursorTo(t, m, "LIDL SOFIA")
|
||||
|
||||
key(t, m, "x")
|
||||
txn := findTxn(t, db, "LIDL SOFIA")
|
||||
if !txn.IsTransfer() {
|
||||
t.Error("expected x to mark the row as a transfer")
|
||||
}
|
||||
|
||||
key(t, m, "x")
|
||||
if findTxn(t, db, "LIDL SOFIA").IsTransfer() {
|
||||
t.Error("expected a second x to unmark it")
|
||||
}
|
||||
|
||||
// c drops the manual override, restoring the rule's verdict.
|
||||
key(t, m, "c")
|
||||
txn = findTxn(t, db, "LIDL SOFIA")
|
||||
if txn.ManualTransfer != nil || txn.ManualTag != "" {
|
||||
t.Errorf("expected c to clear overrides, got manual tag %q transfer %v", txn.ManualTag, txn.ManualTransfer)
|
||||
}
|
||||
if txn.Tag() != "groceries" {
|
||||
t.Errorf("tag after clearing = %q, want groceries from the rule", txn.Tag())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUntaggedToggle(t *testing.T) {
|
||||
m, _ := newTestModel(t)
|
||||
|
||||
key(t, m, "u")
|
||||
if !m.onlyUntagged {
|
||||
t.Fatal("expected u to enable the untagged filter")
|
||||
}
|
||||
if len(m.txns) != 1 || m.txns[0].Description != "SOME RANDOM SHOP" {
|
||||
t.Fatalf("untagged view shows %+v, want only SOME RANDOM SHOP", m.txns)
|
||||
}
|
||||
|
||||
// Tagging the last untagged row empties the view; the cursor must stay valid.
|
||||
cursorTo(t, m, "SOME RANDOM SHOP")
|
||||
key(t, m, "t")
|
||||
typeText(t, m, "hobby")
|
||||
key(t, m, "enter")
|
||||
if len(m.txns) != 0 {
|
||||
t.Errorf("expected the untagged view to be empty, got %d rows", len(m.txns))
|
||||
}
|
||||
if _, ok := m.selected(); ok {
|
||||
t.Error("expected no selection in an empty view")
|
||||
}
|
||||
m.View() // must not panic on an empty table
|
||||
|
||||
key(t, m, "u")
|
||||
if len(m.txns) != 2 {
|
||||
t.Errorf("expected all rows back after toggling off, got %d", len(m.txns))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchFilter(t *testing.T) {
|
||||
m, _ := newTestModel(t)
|
||||
|
||||
key(t, m, "/")
|
||||
if m.input != inputSearch {
|
||||
t.Fatal("expected the search prompt to open")
|
||||
}
|
||||
typeText(t, m, "lidl")
|
||||
key(t, m, "enter")
|
||||
|
||||
if len(m.txns) != 1 || m.txns[0].Description != "LIDL SOFIA" {
|
||||
t.Fatalf("search results = %+v, want only LIDL SOFIA", m.txns)
|
||||
}
|
||||
|
||||
// Reopening the prompt pre-fills the active query so it can be edited.
|
||||
key(t, m, "/")
|
||||
if got := m.text.Value(); got != "lidl" {
|
||||
t.Errorf("search prompt pre-filled with %q, want %q", got, "lidl")
|
||||
}
|
||||
for range len("lidl") {
|
||||
m.Update(tea.KeyMsg{Type: tea.KeyBackspace})
|
||||
}
|
||||
key(t, m, "enter") // an emptied query clears the filter
|
||||
if len(m.txns) != 2 {
|
||||
t.Errorf("expected the filter to clear, got %d rows", len(m.txns))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportViewExcludesTransfers(t *testing.T) {
|
||||
m, _ := newTestModel(t)
|
||||
key(t, m, "2")
|
||||
cursorTo(t, m, "LIDL SOFIA")
|
||||
key(t, m, "x") // mark as a transfer
|
||||
|
||||
key(t, m, "3") // report view
|
||||
view := m.View()
|
||||
if strings.Contains(view, "groceries") {
|
||||
t.Error("a transfer must not appear in the report")
|
||||
}
|
||||
if !strings.Contains(view, "TOTAL") {
|
||||
t.Error("expected a TOTAL row in the report")
|
||||
}
|
||||
}
|
||||
|
||||
func findTxn(t *testing.T, db *store.DB, desc string) model.Transaction {
|
||||
t.Helper()
|
||||
txns, err := db.Transactions(store.Filter{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, txn := range txns {
|
||||
if txn.Description == desc {
|
||||
return txn
|
||||
}
|
||||
}
|
||||
t.Fatalf("no transaction with description %q", desc)
|
||||
return model.Transaction{}
|
||||
}
|
||||
Reference in New Issue
Block a user