// Package store is the SQLite index over the statements. It is entirely // rebuildable: delete index.db and re-import to get it back. Nothing lives // only here. package store import ( "database/sql" "fmt" "os" "path/filepath" "strings" _ "modernc.org/sqlite" "git.petrovv.com/nikola/money/internal/model" ) // DB wraps the SQLite handle. type DB struct { sql *sql.DB } const schema = ` PRAGMA foreign_keys = ON; CREATE TABLE IF NOT EXISTS accounts ( id INTEGER PRIMARY KEY AUTOINCREMENT, slug TEXT NOT NULL UNIQUE, name TEXT NOT NULL, currency TEXT NOT NULL, minor_digits INTEGER NOT NULL DEFAULT 2 ); CREATE TABLE IF NOT EXISTS source_files ( id INTEGER PRIMARY KEY AUTOINCREMENT, account_id INTEGER NOT NULL REFERENCES accounts(id), path TEXT NOT NULL, sha256 TEXT NOT NULL, imported_at TEXT NOT NULL, UNIQUE(account_id, path) ); CREATE TABLE IF NOT EXISTS transactions ( id INTEGER PRIMARY KEY AUTOINCREMENT, account_id INTEGER NOT NULL REFERENCES accounts(id), source_file_id INTEGER NOT NULL REFERENCES source_files(id), fingerprint TEXT NOT NULL, date TEXT NOT NULL, description TEXT NOT NULL, amount_minor INTEGER NOT NULL, type TEXT NOT NULL DEFAULT '', balance_minor INTEGER, rule_tag TEXT, UNIQUE(account_id, fingerprint) ); CREATE INDEX IF NOT EXISTS idx_txn_date ON transactions(date); CREATE INDEX IF NOT EXISTS idx_txn_account ON transactions(account_id); -- One row per matched movement between the user's own accounts, derived from -- the [[transfer]] blocks in rules.toml. A leg belongs to at most one transfer, -- which UNIQUE enforces rather than trusting the pairing to be well behaved. CREATE TABLE IF NOT EXISTS transfers ( id INTEGER PRIMARY KEY AUTOINCREMENT, def_index INTEGER NOT NULL, out_txn_id INTEGER NOT NULL UNIQUE REFERENCES transactions(id), in_txn_id INTEGER NOT NULL UNIQUE REFERENCES transactions(id) ); ` // Open opens (creating if needed) the index at path. // // There is no schema migration: the index caches what the statements and // rules.toml already say, so a build that changes the schema is answered by // deleting index.db and importing again, not by patching the old one in place. // Until it is deleted, queries against it fail on the columns it lacks. func Open(path string) (*DB, error) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return nil, fmt.Errorf("create index dir: %w", err) } sqlDB, err := sql.Open("sqlite", path) if err != nil { return nil, fmt.Errorf("open index %s: %w", path, err) } if _, err := sqlDB.Exec(schema); err != nil { sqlDB.Close() return nil, fmt.Errorf("apply schema: %w", err) } return &DB{sql: sqlDB}, nil } // Close releases the underlying handle. func (d *DB) Close() error { return d.sql.Close() } // UpsertAccount inserts or updates an account by slug and returns its id. func (d *DB) UpsertAccount(a model.Account) (int64, error) { _, err := d.sql.Exec(` INSERT INTO accounts (slug, name, currency, minor_digits) VALUES (?, ?, ?, ?) ON CONFLICT(slug) DO UPDATE SET name = excluded.name, currency = excluded.currency, minor_digits = excluded.minor_digits`, a.Slug, a.Name, a.Currency, a.MinorDigits) if err != nil { return 0, fmt.Errorf("upsert account %s: %w", a.Slug, err) } var id int64 if err := d.sql.QueryRow(`SELECT id FROM accounts WHERE slug = ?`, a.Slug).Scan(&id); err != nil { return 0, fmt.Errorf("read account id %s: %w", a.Slug, err) } return id, nil } // Accounts lists every known account, ordered by slug. func (d *DB) Accounts() ([]model.Account, error) { rows, err := d.sql.Query(`SELECT id, slug, name, currency, minor_digits FROM accounts ORDER BY slug`) if err != nil { return nil, fmt.Errorf("list accounts: %w", err) } defer rows.Close() var out []model.Account for rows.Next() { var a model.Account if err := rows.Scan(&a.ID, &a.Slug, &a.Name, &a.Currency, &a.MinorDigits); err != nil { return nil, err } out = append(out, a) } return out, rows.Err() } // SourceFile records that a statement file was imported, returning its id. func (d *DB) SourceFile(accountID int64, path, sha, importedAt string) (int64, error) { _, err := d.sql.Exec(` INSERT INTO source_files (account_id, path, sha256, imported_at) VALUES (?, ?, ?, ?) ON CONFLICT(account_id, path) DO UPDATE SET sha256 = excluded.sha256, imported_at = excluded.imported_at`, accountID, path, sha, importedAt) if err != nil { return 0, fmt.Errorf("record source file %s: %w", path, err) } var id int64 if err := d.sql.QueryRow( `SELECT id FROM source_files WHERE account_id = ? AND path = ?`, accountID, path).Scan(&id); err != nil { return 0, fmt.Errorf("read source file id %s: %w", path, err) } return id, nil } // SourceFileSHA returns the recorded checksum for a statement file, and whether // it has been imported before. func (d *DB) SourceFileSHA(accountID int64, path string) (string, bool, error) { var sha string err := d.sql.QueryRow( `SELECT sha256 FROM source_files WHERE account_id = ? AND path = ?`, accountID, path).Scan(&sha) if err == sql.ErrNoRows { return "", false, nil } if err != nil { return "", false, err } return sha, true, nil } // InsertTransaction adds a transaction unless its fingerprint already exists // for that account. It reports whether a new row was created. // // Existing rows are deliberately left untouched, so re-importing a statement // that overlaps one already imported adds nothing rather than duplicating it. func (d *DB) InsertTransaction(t model.Transaction) (bool, error) { var balance any if t.BalanceMinor != nil { balance = *t.BalanceMinor } res, err := d.sql.Exec(` INSERT INTO transactions (account_id, source_file_id, fingerprint, date, description, amount_minor, type, balance_minor, rule_tag) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULLIF(?, '')) ON CONFLICT(account_id, fingerprint) DO NOTHING`, t.AccountID, t.SourceFileID, t.Fingerprint, t.Date, t.Description, t.AmountMinor, t.Type, balance, t.RuleTag) if err != nil { return false, fmt.Errorf("insert transaction: %w", err) } n, err := res.RowsAffected() if err != nil { return false, err } return n > 0, nil } // Filter narrows a transaction query. type Filter struct { AccountSlug string // Untagged selects the rows still waiting for a verdict: no tag, and not a // leg of a matched transfer. A paired leg has been accounted for by the // transfer that claimed it, so listing it as untagged would ask the user to // write a rule for something that is already spoken for and that the report // leaves out anyway. Untagged bool Month string // YYYY-MM // From and To bound the dates inclusively, as YYYY-MM-DD, either side empty // for unbounded. Dates are stored ISO-8601, so a string comparison is a date // comparison; nothing has to be parsed to filter on a range. From string To string Search string // case-insensitive substring of the description Limit int } // Transactions returns rows matching f, newest first. func (d *DB) Transactions(f Filter) ([]model.Transaction, error) { q := ` SELECT t.id, t.account_id, a.slug, a.currency, a.minor_digits, t.fingerprint, t.date, t.description, t.amount_minor, COALESCE(s.path, ''), t.type, t.balance_minor, COALESCE(t.rule_tag, ''), x.id FROM transactions t JOIN accounts a ON a.id = t.account_id LEFT JOIN source_files s ON s.id = t.source_file_id LEFT JOIN transfers x ON x.out_txn_id = t.id OR x.in_txn_id = t.id WHERE 1 = 1` var args []any if f.AccountSlug != "" { q += ` AND a.slug = ?` args = append(args, f.AccountSlug) } if f.Untagged { q += ` AND NULLIF(t.rule_tag, '') IS NULL AND x.id IS NULL` } if f.Month != "" { q += ` AND substr(t.date, 1, 7) = ?` args = append(args, f.Month) } if f.From != "" { q += ` AND t.date >= ?` args = append(args, f.From) } if f.To != "" { q += ` AND t.date <= ?` args = append(args, f.To) } q += ` ORDER BY t.date DESC, t.id DESC` // Search and Limit are applied in Go: SQLite's upper()/LIKE fold ASCII // only, which would silently fail on Cyrillic statement descriptions. rows, err := d.sql.Query(q, args...) if err != nil { return nil, fmt.Errorf("query transactions: %w", err) } defer rows.Close() needle := model.NormalizeDescription(f.Search) var out []model.Transaction for rows.Next() { var ( t model.Transaction balance sql.NullInt64 transfer sql.NullInt64 ) if err := rows.Scan(&t.ID, &t.AccountID, &t.AccountSlug, &t.Currency, &t.MinorDigits, &t.Fingerprint, &t.Date, &t.Description, &t.AmountMinor, &t.SourcePath, &t.Type, &balance, &t.RuleTag, &transfer); err != nil { return nil, err } if balance.Valid { v := balance.Int64 t.BalanceMinor = &v } if transfer.Valid { v := transfer.Int64 t.TransferID = &v } if needle != "" && !strings.Contains(model.NormalizeDescription(t.Description), needle) { continue } out = append(out, t) if f.Limit > 0 && len(out) >= f.Limit { break } } return out, rows.Err() } // RuleAssignment is one row's recomputed rule verdict. type RuleAssignment struct { ID int64 Tag string } // ApplyRuleResults rewrites rule_tag for every listed row in a single // transaction. The manual column is never touched. func (d *DB) ApplyRuleResults(rs []RuleAssignment) error { tx, err := d.sql.Begin() if err != nil { return err } defer tx.Rollback() stmt, err := tx.Prepare( `UPDATE transactions SET rule_tag = NULLIF(?, '') WHERE id = ?`) if err != nil { return err } defer stmt.Close() for _, r := range rs { if _, err := stmt.Exec(r.Tag, r.ID); err != nil { return fmt.Errorf("apply rules to txn %d: %w", r.ID, err) } } return tx.Commit() } // TransferLink is one matched pair, as decided by the transfer definitions. type TransferLink struct { DefIndex int OutID int64 InID int64 } // ReplaceTransfers rewrites the whole pairing in one transaction. Like the // tags, it is derived from a file the user edits, so it is replaced wholesale // rather than patched: a definition removed from rules.toml must take its pairs // with it. func (d *DB) ReplaceTransfers(links []TransferLink) error { tx, err := d.sql.Begin() if err != nil { return err } defer tx.Rollback() if _, err := tx.Exec(`DELETE FROM transfers`); err != nil { return fmt.Errorf("clear transfers: %w", err) } stmt, err := tx.Prepare( `INSERT INTO transfers (def_index, out_txn_id, in_txn_id) VALUES (?, ?, ?)`) if err != nil { return err } defer stmt.Close() for _, l := range links { if _, err := stmt.Exec(l.DefIndex, l.OutID, l.InID); err != nil { return fmt.Errorf("link transfer %d→%d: %w", l.OutID, l.InID, err) } } return tx.Commit() } // Balance sums every transaction in an account. func (d *DB) Balance(accountID int64) (int64, error) { var v sql.NullInt64 err := d.sql.QueryRow( `SELECT SUM(amount_minor) FROM transactions WHERE account_id = ?`, accountID).Scan(&v) if err != nil { return 0, err } return v.Int64, nil } // Count returns the number of transactions in an account. func (d *DB) Count(accountID int64) (int, error) { var n int err := d.sql.QueryRow( `SELECT COUNT(*) FROM transactions WHERE account_id = ?`, accountID).Scan(&n) return n, err } // Months lists the distinct YYYY-MM the index holds, most recent first. It is // asked of the whole index rather than of a filtered view on purpose: it is the // time axis the report screen steps along, and an axis that grew and shrank as // the account or search filter changed would move under the cursor. func (d *DB) Months() ([]string, error) { rows, err := d.sql.Query( `SELECT DISTINCT substr(date, 1, 7) FROM transactions ORDER BY 1 DESC`) if err != nil { return nil, fmt.Errorf("list months: %w", err) } defer rows.Close() var out []string for rows.Next() { var s string if err := rows.Scan(&s); err != nil { return nil, err } out = append(out, s) } return out, rows.Err() } // Tags lists every tag in use, for completion in the rule builder. func (d *DB) Tags() ([]string, error) { rows, err := d.sql.Query(` SELECT DISTINCT rule_tag FROM transactions WHERE rule_tag IS NOT NULL AND rule_tag != '' ORDER BY rule_tag`) if err != nil { return nil, err } defer rows.Close() var out []string for rows.Next() { var s string if err := rows.Scan(&s); err != nil { return nil, err } out = append(out, s) } return out, rows.Err() }