// Package store is the SQLite index over the statements. It is entirely // rebuildable: delete index.db and re-import to get it back, except for the // tags set by hand, which live only here. package store import ( "database/sql" "fmt" "os" "path/filepath" "strings" _ "modernc.org/sqlite" "git.petrovv.com/nikola/money/internal/model" ) // DB wraps the SQLite handle. type DB struct { sql *sql.DB } const schema = ` PRAGMA foreign_keys = ON; CREATE TABLE IF NOT EXISTS accounts ( id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT NOT NULL UNIQUE, name TEXT NOT NULL, currency TEXT NOT NULL, minor_digits INTEGER NOT NULL DEFAULT 2 ); CREATE TABLE IF NOT EXISTS source_files ( id INTEGER PRIMARY KEY AUTOINCREMENT, account_id INTEGER NOT NULL REFERENCES accounts(id), path TEXT NOT NULL, sha256 TEXT NOT NULL, imported_at TEXT NOT NULL, UNIQUE(account_id, path) ); CREATE TABLE IF NOT EXISTS transactions ( id INTEGER PRIMARY KEY AUTOINCREMENT, account_id INTEGER NOT NULL REFERENCES accounts(id), source_file_id INTEGER NOT NULL REFERENCES source_files(id), fingerprint TEXT NOT NULL, date TEXT NOT NULL, description TEXT NOT NULL, amount_minor INTEGER NOT NULL, type TEXT NOT NULL DEFAULT '', balance_minor INTEGER, rule_tag TEXT, UNIQUE(account_id, fingerprint) ); CREATE INDEX IF NOT EXISTS idx_txn_date ON transactions(date); CREATE INDEX IF NOT EXISTS idx_txn_account ON transactions(account_id); ` // migrations bring an index created by an older build up to date. SQLite // errors on a duplicate column, which is how we detect "already applied". var migrations = []string{ `ALTER TABLE transactions ADD COLUMN type TEXT NOT NULL DEFAULT ''`, `ALTER TABLE transactions ADD COLUMN balance_minor INTEGER`, } // dropped are columns an older build created that this one no longer reads. // Nothing indexes or constrains them, so they can simply go; leaving them // would keep a NOT NULL column alive that no INSERT here ever names. var dropped = []string{"counterparty", "rule_transfer", "manual_transfer", "manual_tag"} // migrate brings an index created by an older build up to date, adding the // columns it lacks and removing the ones it should no longer have. func migrate(db *sql.DB) error { have, err := columns(db, "transactions") if err != nil { return err } for _, stmt := range migrations { name := addedColumn(stmt) if have[name] { continue } if _, err := db.Exec(stmt); err != nil { return fmt.Errorf("migrate (%s): %w", stmt, err) } } for _, name := range dropped { if !have[name] { continue } stmt := fmt.Sprintf(`ALTER TABLE transactions DROP COLUMN %s`, name) if _, err := db.Exec(stmt); err != nil { return fmt.Errorf("migrate (%s): %w", stmt, err) } } return nil } func columns(db *sql.DB, table string) (map[string]bool, error) { rows, err := db.Query(`SELECT name FROM pragma_table_info(?)`, table) if err != nil { return nil, fmt.Errorf("inspect %s: %w", table, err) } defer rows.Close() out := map[string]bool{} for rows.Next() { var name string if err := rows.Scan(&name); err != nil { return nil, err } out[name] = true } return out, rows.Err() } // addedColumn pulls the column name out of an ADD COLUMN statement. func addedColumn(stmt string) string { _, rest, ok := strings.Cut(stmt, "ADD COLUMN ") if !ok { return "" } name, _, _ := strings.Cut(rest, " ") return name } // Open opens (creating if needed) the index at path. func Open(path string) (*DB, error) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return nil, fmt.Errorf("create index dir: %w", err) } sqlDB, err := sql.Open("sqlite", path) if err != nil { return nil, fmt.Errorf("open index %s: %w", path, err) } if _, err := sqlDB.Exec(schema); err != nil { sqlDB.Close() return nil, fmt.Errorf("apply schema: %w", err) } if err := migrate(sqlDB); err != nil { sqlDB.Close() return nil, err } return &DB{sql: sqlDB}, nil } // Close releases the underlying handle. func (d *DB) Close() error { return d.sql.Close() } // UpsertAccount inserts or updates an account by slug and returns its id. func (d *DB) UpsertAccount(a model.Account) (int64, error) { _, err := d.sql.Exec(` INSERT INTO accounts (slug, name, currency, minor_digits) VALUES (?, ?, ?, ?) ON CONFLICT(slug) DO UPDATE SET name = excluded.name, currency = excluded.currency, minor_digits = excluded.minor_digits`, a.Slug, a.Name, a.Currency, a.MinorDigits) if err != nil { return 0, fmt.Errorf("upsert account %s: %w", a.Slug, err) } var id int64 if err := d.sql.QueryRow(`SELECT id FROM accounts WHERE slug = ?`, a.Slug).Scan(&id); err != nil { return 0, fmt.Errorf("read account id %s: %w", a.Slug, err) } return id, nil } // Accounts lists every known account, ordered by slug. func (d *DB) Accounts() ([]model.Account, error) { rows, err := d.sql.Query(`SELECT id, slug, name, currency, minor_digits FROM accounts ORDER BY slug`) if err != nil { return nil, fmt.Errorf("list accounts: %w", err) } defer rows.Close() var out []model.Account for rows.Next() { var a model.Account if err := rows.Scan(&a.ID, &a.Slug, &a.Name, &a.Currency, &a.MinorDigits); err != nil { return nil, err } out = append(out, a) } return out, rows.Err() } // SourceFile records that a statement file was imported, returning its id. func (d *DB) SourceFile(accountID int64, path, sha, importedAt string) (int64, error) { _, err := d.sql.Exec(` INSERT INTO source_files (account_id, path, sha256, imported_at) VALUES (?, ?, ?, ?) ON CONFLICT(account_id, path) DO UPDATE SET sha256 = excluded.sha256, imported_at = excluded.imported_at`, accountID, path, sha, importedAt) if err != nil { return 0, fmt.Errorf("record source file %s: %w", path, err) } var id int64 if err := d.sql.QueryRow( `SELECT id FROM source_files WHERE account_id = ? AND path = ?`, accountID, path).Scan(&id); err != nil { return 0, fmt.Errorf("read source file id %s: %w", path, err) } return id, nil } // SourceFileSHA returns the recorded checksum for a statement file, and whether // it has been imported before. func (d *DB) SourceFileSHA(accountID int64, path string) (string, bool, error) { var sha string err := d.sql.QueryRow( `SELECT sha256 FROM source_files WHERE account_id = ? AND path = ?`, accountID, path).Scan(&sha) if err == sql.ErrNoRows { return "", false, nil } if err != nil { return "", false, err } return sha, true, nil } // InsertTransaction adds a transaction unless its fingerprint already exists // for that account. It reports whether a new row was created. // // Existing rows are deliberately left untouched, so re-importing a statement // that overlaps one already imported adds nothing rather than duplicating it. func (d *DB) InsertTransaction(t model.Transaction) (bool, error) { var balance any if t.BalanceMinor != nil { balance = *t.BalanceMinor } res, err := d.sql.Exec(` INSERT INTO transactions (account_id, source_file_id, fingerprint, date, description, amount_minor, type, balance_minor, rule_tag) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULLIF(?, '')) ON CONFLICT(account_id, fingerprint) DO NOTHING`, t.AccountID, t.SourceFileID, t.Fingerprint, t.Date, t.Description, t.AmountMinor, t.Type, balance, t.RuleTag) if err != nil { return false, fmt.Errorf("insert transaction: %w", err) } n, err := res.RowsAffected() if err != nil { return false, err } return n > 0, nil } // Filter narrows a transaction query. type Filter struct { AccountSlug string Untagged bool // only rows with no effective tag Month string // YYYY-MM Search string // case-insensitive substring of the description Limit int } // Transactions returns rows matching f, newest first. func (d *DB) Transactions(f Filter) ([]model.Transaction, error) { q := ` SELECT t.id, t.account_id, a.slug, a.currency, a.minor_digits, t.fingerprint, t.date, t.description, t.amount_minor, COALESCE(s.path, ''), t.type, t.balance_minor, COALESCE(t.rule_tag, '') FROM transactions t JOIN accounts a ON a.id = t.account_id LEFT JOIN source_files s ON s.id = t.source_file_id WHERE 1 = 1` var args []any if f.AccountSlug != "" { q += ` AND a.slug = ?` args = append(args, f.AccountSlug) } if f.Untagged { q += ` AND NULLIF(t.rule_tag, '') IS NULL` } if f.Month != "" { q += ` AND substr(t.date, 1, 7) = ?` args = append(args, f.Month) } q += ` ORDER BY t.date DESC, t.id DESC` // Search and Limit are applied in Go: SQLite's upper()/LIKE fold ASCII // only, which would silently fail on Cyrillic statement descriptions. rows, err := d.sql.Query(q, args...) if err != nil { return nil, fmt.Errorf("query transactions: %w", err) } defer rows.Close() needle := model.NormalizeDescription(f.Search) var out []model.Transaction for rows.Next() { var ( t model.Transaction balance sql.NullInt64 ) if err := rows.Scan(&t.ID, &t.AccountID, &t.AccountSlug, &t.Currency, &t.MinorDigits, &t.Fingerprint, &t.Date, &t.Description, &t.AmountMinor, &t.SourcePath, &t.Type, &balance, &t.RuleTag); err != nil { return nil, err } if balance.Valid { v := balance.Int64 t.BalanceMinor = &v } if needle != "" && !strings.Contains(model.NormalizeDescription(t.Description), needle) { continue } out = append(out, t) if f.Limit > 0 && len(out) >= f.Limit { break } } return out, rows.Err() } // RuleAssignment is one row's recomputed rule verdict. type RuleAssignment struct { ID int64 Tag string } // ApplyRuleResults rewrites rule_tag for every listed row in a single // transaction. The manual column is never touched. func (d *DB) ApplyRuleResults(rs []RuleAssignment) error { tx, err := d.sql.Begin() if err != nil { return err } defer tx.Rollback() stmt, err := tx.Prepare( `UPDATE transactions SET rule_tag = NULLIF(?, '') WHERE id = ?`) if err != nil { return err } defer stmt.Close() for _, r := range rs { if _, err := stmt.Exec(r.Tag, r.ID); err != nil { return fmt.Errorf("apply rules to txn %d: %w", r.ID, err) } } return tx.Commit() } // Balance sums every transaction in an account. func (d *DB) Balance(accountID int64) (int64, error) { var v sql.NullInt64 err := d.sql.QueryRow( `SELECT SUM(amount_minor) FROM transactions WHERE account_id = ?`, accountID).Scan(&v) if err != nil { return 0, err } return v.Int64, nil } // Count returns the number of transactions in an account. func (d *DB) Count(accountID int64) (int, error) { var n int err := d.sql.QueryRow( `SELECT COUNT(*) FROM transactions WHERE account_id = ?`, accountID).Scan(&n) return n, err } // Tags lists every tag in use, for completion in the TUI. func (d *DB) Tags() ([]string, error) { rows, err := d.sql.Query(` SELECT DISTINCT rule_tag FROM transactions WHERE rule_tag IS NOT NULL AND rule_tag != '' ORDER BY rule_tag`) if err != nil { return nil, err } defer rows.Close() var out []string for rows.Next() { var s string if err := rows.Scan(&s); err != nil { return nil, err } out = append(out, s) } return out, rows.Err() }