Add money: statement-driven personal finance tracker
A data directory holds one folder per account. Statements dropped into those folders are parsed into a rebuildable SQLite index, categorised by ordered glob rules in rules.toml, and browsed or hand-tagged in a Bubble Tea TUI. Movements between the user's own accounts are marked as transfers by the same rules and excluded from spending totals. Manual tags and transfer marks are stored separately from the rule-derived ones and always win, so editing rules.toml and re-running retag never destroys hand edits. Parsers are pluggable. Three are ported from the Python extractors they replace -- nlb and traderepublic read PDFs via pdftotext -layout, revolut reads the CSV export -- alongside a configurable-column CSV parser and a cmd parser that shells out to an external script. Both ports fix two latent bugs in the originals: the sign character class rejected the typographic minus U+2212 that some PDF fonts emit, and NLB's hardcoded continuation indent broke when pdftotext compressed runs of spaces, so the threshold is now measured from the description column. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,420 @@
|
||||
// Package store is the SQLite index over the statements. It is entirely
|
||||
// rebuildable: delete .money/index.db and re-import to get it back, except for
|
||||
// manual tags and manual transfer overrides, which live only here.
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"git.petrovv.com/nikola/money/internal/model"
|
||||
)
|
||||
|
||||
// DB wraps the SQLite handle.
|
||||
type DB struct {
|
||||
sql *sql.DB
|
||||
}
|
||||
|
||||
const schema = `
|
||||
PRAGMA foreign_keys = ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS accounts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
currency TEXT NOT NULL,
|
||||
minor_digits INTEGER NOT NULL DEFAULT 2
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS source_files (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
account_id INTEGER NOT NULL REFERENCES accounts(id),
|
||||
path TEXT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
imported_at TEXT NOT NULL,
|
||||
UNIQUE(account_id, path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS transactions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
account_id INTEGER NOT NULL REFERENCES accounts(id),
|
||||
source_file_id INTEGER NOT NULL REFERENCES source_files(id),
|
||||
fingerprint TEXT NOT NULL,
|
||||
date TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
amount_minor INTEGER NOT NULL,
|
||||
counterparty TEXT NOT NULL DEFAULT '',
|
||||
type TEXT NOT NULL DEFAULT '',
|
||||
balance_minor INTEGER,
|
||||
rule_tag TEXT,
|
||||
manual_tag TEXT,
|
||||
rule_transfer INTEGER NOT NULL DEFAULT 0,
|
||||
manual_transfer INTEGER,
|
||||
UNIQUE(account_id, fingerprint)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_txn_date ON transactions(date);
|
||||
CREATE INDEX IF NOT EXISTS idx_txn_account ON transactions(account_id);
|
||||
`
|
||||
|
||||
// migrations bring an index created by an older build up to date. SQLite
|
||||
// errors on a duplicate column, which is how we detect "already applied".
|
||||
var migrations = []string{
|
||||
`ALTER TABLE transactions ADD COLUMN counterparty TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE transactions ADD COLUMN type TEXT NOT NULL DEFAULT ''`,
|
||||
`ALTER TABLE transactions ADD COLUMN balance_minor INTEGER`,
|
||||
}
|
||||
|
||||
// migrate applies any column that this index is missing.
|
||||
func migrate(db *sql.DB) error {
|
||||
have, err := columns(db, "transactions")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, stmt := range migrations {
|
||||
name := addedColumn(stmt)
|
||||
if have[name] {
|
||||
continue
|
||||
}
|
||||
if _, err := db.Exec(stmt); err != nil {
|
||||
return fmt.Errorf("migrate (%s): %w", stmt, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func columns(db *sql.DB, table string) (map[string]bool, error) {
|
||||
rows, err := db.Query(`SELECT name FROM pragma_table_info(?)`, table)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("inspect %s: %w", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// addedColumn pulls the column name out of an ADD COLUMN statement.
|
||||
func addedColumn(stmt string) string {
|
||||
_, rest, ok := strings.Cut(stmt, "ADD COLUMN ")
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
name, _, _ := strings.Cut(rest, " ")
|
||||
return name
|
||||
}
|
||||
|
||||
// Open opens (creating if needed) the index at path.
|
||||
func Open(path string) (*DB, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return nil, fmt.Errorf("create state dir: %w", err)
|
||||
}
|
||||
sqlDB, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open index %s: %w", path, err)
|
||||
}
|
||||
if _, err := sqlDB.Exec(schema); err != nil {
|
||||
sqlDB.Close()
|
||||
return nil, fmt.Errorf("apply schema: %w", err)
|
||||
}
|
||||
if err := migrate(sqlDB); err != nil {
|
||||
sqlDB.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &DB{sql: sqlDB}, nil
|
||||
}
|
||||
|
||||
// Close releases the underlying handle.
|
||||
func (d *DB) Close() error { return d.sql.Close() }
|
||||
|
||||
// UpsertAccount inserts or updates an account by slug and returns its id.
|
||||
func (d *DB) UpsertAccount(a model.Account) (int64, error) {
|
||||
_, err := d.sql.Exec(`
|
||||
INSERT INTO accounts (slug, name, currency, minor_digits)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(slug) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
currency = excluded.currency,
|
||||
minor_digits = excluded.minor_digits`,
|
||||
a.Slug, a.Name, a.Currency, a.MinorDigits)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("upsert account %s: %w", a.Slug, err)
|
||||
}
|
||||
var id int64
|
||||
if err := d.sql.QueryRow(`SELECT id FROM accounts WHERE slug = ?`, a.Slug).Scan(&id); err != nil {
|
||||
return 0, fmt.Errorf("read account id %s: %w", a.Slug, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Accounts lists every known account, ordered by slug.
|
||||
func (d *DB) Accounts() ([]model.Account, error) {
|
||||
rows, err := d.sql.Query(`SELECT id, slug, name, currency, minor_digits FROM accounts ORDER BY slug`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list accounts: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []model.Account
|
||||
for rows.Next() {
|
||||
var a model.Account
|
||||
if err := rows.Scan(&a.ID, &a.Slug, &a.Name, &a.Currency, &a.MinorDigits); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SourceFile records that a statement file was imported, returning its id.
|
||||
func (d *DB) SourceFile(accountID int64, path, sha, importedAt string) (int64, error) {
|
||||
_, err := d.sql.Exec(`
|
||||
INSERT INTO source_files (account_id, path, sha256, imported_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(account_id, path) DO UPDATE SET
|
||||
sha256 = excluded.sha256,
|
||||
imported_at = excluded.imported_at`,
|
||||
accountID, path, sha, importedAt)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("record source file %s: %w", path, err)
|
||||
}
|
||||
var id int64
|
||||
if err := d.sql.QueryRow(
|
||||
`SELECT id FROM source_files WHERE account_id = ? AND path = ?`, accountID, path).Scan(&id); err != nil {
|
||||
return 0, fmt.Errorf("read source file id %s: %w", path, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// SourceFileSHA returns the recorded checksum for a statement file, and whether
|
||||
// it has been imported before.
|
||||
func (d *DB) SourceFileSHA(accountID int64, path string) (string, bool, error) {
|
||||
var sha string
|
||||
err := d.sql.QueryRow(
|
||||
`SELECT sha256 FROM source_files WHERE account_id = ? AND path = ?`, accountID, path).Scan(&sha)
|
||||
if err == sql.ErrNoRows {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return sha, true, nil
|
||||
}
|
||||
|
||||
// InsertTransaction adds a transaction unless its fingerprint already exists
|
||||
// for that account. It reports whether a new row was created.
|
||||
//
|
||||
// Existing rows are deliberately left untouched: re-importing an overlapping
|
||||
// statement must never clobber tags the user set by hand.
|
||||
func (d *DB) InsertTransaction(t model.Transaction) (bool, error) {
|
||||
var balance any
|
||||
if t.BalanceMinor != nil {
|
||||
balance = *t.BalanceMinor
|
||||
}
|
||||
res, err := d.sql.Exec(`
|
||||
INSERT INTO transactions
|
||||
(account_id, source_file_id, fingerprint, date, description, amount_minor,
|
||||
counterparty, type, balance_minor, rule_tag, rule_transfer)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULLIF(?, ''), ?)
|
||||
ON CONFLICT(account_id, fingerprint) DO NOTHING`,
|
||||
t.AccountID, t.SourceFileID, t.Fingerprint, t.Date, t.Description,
|
||||
t.AmountMinor, t.Counterparty, t.Type, balance, t.RuleTag, boolToInt(t.RuleTransfer))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("insert transaction: %w", err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// Filter narrows a transaction query.
|
||||
type Filter struct {
|
||||
AccountSlug string
|
||||
Untagged bool // only rows with no effective tag
|
||||
Month string // YYYY-MM
|
||||
Search string // case-insensitive substring of the description
|
||||
Limit int
|
||||
}
|
||||
|
||||
// Transactions returns rows matching f, newest first.
|
||||
func (d *DB) Transactions(f Filter) ([]model.Transaction, error) {
|
||||
q := `
|
||||
SELECT t.id, t.account_id, a.slug, a.currency, a.minor_digits,
|
||||
t.fingerprint, t.date, t.description, t.amount_minor,
|
||||
COALESCE(s.path, ''), t.counterparty, t.type, t.balance_minor,
|
||||
COALESCE(t.rule_tag, ''), COALESCE(t.manual_tag, ''),
|
||||
t.rule_transfer, t.manual_transfer
|
||||
FROM transactions t
|
||||
JOIN accounts a ON a.id = t.account_id
|
||||
LEFT JOIN source_files s ON s.id = t.source_file_id
|
||||
WHERE 1 = 1`
|
||||
var args []any
|
||||
if f.AccountSlug != "" {
|
||||
q += ` AND a.slug = ?`
|
||||
args = append(args, f.AccountSlug)
|
||||
}
|
||||
if f.Untagged {
|
||||
q += ` AND COALESCE(NULLIF(t.manual_tag, ''), NULLIF(t.rule_tag, '')) IS NULL`
|
||||
}
|
||||
if f.Month != "" {
|
||||
q += ` AND substr(t.date, 1, 7) = ?`
|
||||
args = append(args, f.Month)
|
||||
}
|
||||
q += ` ORDER BY t.date DESC, t.id DESC`
|
||||
// Search and Limit are applied in Go: SQLite's upper()/LIKE fold ASCII
|
||||
// only, which would silently fail on Cyrillic statement descriptions.
|
||||
|
||||
rows, err := d.sql.Query(q, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query transactions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
needle := model.NormalizeDescription(f.Search)
|
||||
var out []model.Transaction
|
||||
for rows.Next() {
|
||||
var (
|
||||
t model.Transaction
|
||||
ruleTransfer int
|
||||
manualTransfer sql.NullInt64
|
||||
balance sql.NullInt64
|
||||
)
|
||||
if err := rows.Scan(&t.ID, &t.AccountID, &t.AccountSlug, &t.Currency, &t.MinorDigits,
|
||||
&t.Fingerprint, &t.Date, &t.Description, &t.AmountMinor, &t.SourcePath,
|
||||
&t.Counterparty, &t.Type, &balance,
|
||||
&t.RuleTag, &t.ManualTag, &ruleTransfer, &manualTransfer); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.RuleTransfer = ruleTransfer != 0
|
||||
if balance.Valid {
|
||||
v := balance.Int64
|
||||
t.BalanceMinor = &v
|
||||
}
|
||||
if manualTransfer.Valid {
|
||||
v := manualTransfer.Int64 != 0
|
||||
t.ManualTransfer = &v
|
||||
}
|
||||
if needle != "" && !strings.Contains(model.NormalizeDescription(t.Description), needle) {
|
||||
continue
|
||||
}
|
||||
out = append(out, t)
|
||||
if f.Limit > 0 && len(out) >= f.Limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SetManualTag sets (or, with an empty tag, clears) the user's tag override.
|
||||
func (d *DB) SetManualTag(id int64, tag string) error {
|
||||
_, err := d.sql.Exec(`UPDATE transactions SET manual_tag = NULLIF(?, '') WHERE id = ?`, tag, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// SetManualTransfer sets the user's transfer override; nil restores the
|
||||
// rule-derived value.
|
||||
func (d *DB) SetManualTransfer(id int64, v *bool) error {
|
||||
var arg any
|
||||
if v != nil {
|
||||
arg = boolToInt(*v)
|
||||
}
|
||||
_, err := d.sql.Exec(`UPDATE transactions SET manual_transfer = ? WHERE id = ?`, arg, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// ClearOverrides drops both manual overrides for a transaction.
|
||||
func (d *DB) ClearOverrides(id int64) error {
|
||||
_, err := d.sql.Exec(
|
||||
`UPDATE transactions SET manual_tag = NULL, manual_transfer = NULL WHERE id = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// RuleAssignment is one row's recomputed rule verdict.
|
||||
type RuleAssignment struct {
|
||||
ID int64
|
||||
Tag string
|
||||
Transfer bool
|
||||
}
|
||||
|
||||
// ApplyRuleResults rewrites rule_tag and rule_transfer for every listed row in
|
||||
// a single transaction. Manual columns are never touched.
|
||||
func (d *DB) ApplyRuleResults(rs []RuleAssignment) error {
|
||||
tx, err := d.sql.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
stmt, err := tx.Prepare(
|
||||
`UPDATE transactions SET rule_tag = NULLIF(?, ''), rule_transfer = ? WHERE id = ?`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
for _, r := range rs {
|
||||
if _, err := stmt.Exec(r.Tag, boolToInt(r.Transfer), r.ID); err != nil {
|
||||
return fmt.Errorf("apply rules to txn %d: %w", r.ID, err)
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// Balance sums every transaction in an account, transfers included: a transfer
|
||||
// really does move the account's money.
|
||||
func (d *DB) Balance(accountID int64) (int64, error) {
|
||||
var v sql.NullInt64
|
||||
err := d.sql.QueryRow(
|
||||
`SELECT SUM(amount_minor) FROM transactions WHERE account_id = ?`, accountID).Scan(&v)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return v.Int64, nil
|
||||
}
|
||||
|
||||
// Count returns the number of transactions in an account.
|
||||
func (d *DB) Count(accountID int64) (int, error) {
|
||||
var n int
|
||||
err := d.sql.QueryRow(
|
||||
`SELECT COUNT(*) FROM transactions WHERE account_id = ?`, accountID).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Tags lists every effective tag in use, for completion in the TUI.
|
||||
func (d *DB) Tags() ([]string, error) {
|
||||
rows, err := d.sql.Query(`
|
||||
SELECT DISTINCT COALESCE(NULLIF(manual_tag, ''), NULLIF(rule_tag, '')) AS tag
|
||||
FROM transactions WHERE tag IS NOT NULL ORDER BY tag`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var s string
|
||||
if err := rows.Scan(&s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func boolToInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
Reference in New Issue
Block a user