Name NLB uploads, delete statements, and stop importing on upload
Four changes to statement handling in the web app, made together and touching the same upload and statements-list code. Name NLB uploads by statement date. parser.Namer is an optional interface, like Warner, through which a parser names its statements; nlb reads the "Datum izpiska" from the izpisek header and names it izpisek_YYYY_MM_DD, lowercase, extension included -- ported from the rename_izpiski.py it replaces. Uploads are staged as dotfiles, invisible to import, so the parser can read them; two downloads of one statement then meet under one name and the second is recognised as already there, while a different statement of the same date is numbered _2 as the script did. Only uploads are named: source_files records statements by path, so renaming a file already in a folder would orphan its rows. Delete a statement from the statements list. The file is removed from disk for good -- the page says so before it asks -- and store.ForgetSourceFile drops its transactions and their transfer rows. A row two overlapping statements share is stored once, under the file imported first, so it goes too; the account's other statements forget their checksums and show as changed until the next Import re-reads them and restores it. A file already gone from disk can be forgotten. Upload and delete no longer import. Importing stays the user's call, made with the Import button, so a batch can be put together and looked over first. Delete still re-pairs transfers, which reads no statement. Show rows and new rows per statement. The list read "0" for a file whose rows an earlier, overlapping statement already held, which looked like a file that failed to parse. source_files now records how many transactions each statement holds, and the list reads "3 rows · 0 new". This adds a column the code reads, so an index built by an earlier version fails with "no such column: s.rows": delete index.db and import again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+64
-8
@@ -37,6 +37,9 @@ CREATE TABLE IF NOT EXISTS source_files (
|
||||
path TEXT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
imported_at TEXT NOT NULL,
|
||||
-- How many transactions the statement holds, as parsed. Not how many it
|
||||
-- brought in: rows an overlapping statement already had are deduplicated.
|
||||
rows INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE(account_id, path)
|
||||
);
|
||||
|
||||
@@ -130,15 +133,17 @@ func (d *DB) Accounts() ([]model.Account, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// SourceFile records that a statement file was imported, returning its id.
|
||||
func (d *DB) SourceFile(accountID int64, path, sha, importedAt string) (int64, error) {
|
||||
// SourceFile records that a statement file was imported, holding rows
|
||||
// transactions, and returns its id.
|
||||
func (d *DB) SourceFile(accountID int64, path, sha, importedAt string, rows int) (int64, error) {
|
||||
_, err := d.sql.Exec(`
|
||||
INSERT INTO source_files (account_id, path, sha256, imported_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
INSERT INTO source_files (account_id, path, sha256, imported_at, rows)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(account_id, path) DO UPDATE SET
|
||||
sha256 = excluded.sha256,
|
||||
imported_at = excluded.imported_at`,
|
||||
accountID, path, sha, importedAt)
|
||||
imported_at = excluded.imported_at,
|
||||
rows = excluded.rows`,
|
||||
accountID, path, sha, importedAt, rows)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("record source file %s: %w", path, err)
|
||||
}
|
||||
@@ -399,6 +404,8 @@ type SourceFileInfo struct {
|
||||
Path string // relative to the data root, as the importer records it
|
||||
SHA256 string
|
||||
ImportedAt string
|
||||
// Rows is how many transactions the statement holds, as parsed.
|
||||
Rows int
|
||||
// Added counts the transactions this file introduced. A row seen again in
|
||||
// an overlapping statement is deduplicated and stays with the file that
|
||||
// brought it first, so this is not how many rows the file holds.
|
||||
@@ -408,7 +415,7 @@ type SourceFileInfo struct {
|
||||
// SourceFiles lists every statement the index has imported.
|
||||
func (d *DB) SourceFiles() ([]SourceFileInfo, error) {
|
||||
rows, err := d.sql.Query(`
|
||||
SELECT a.slug, s.path, s.sha256, s.imported_at, COUNT(t.id)
|
||||
SELECT a.slug, s.path, s.sha256, s.imported_at, s.rows, COUNT(t.id)
|
||||
FROM source_files s
|
||||
JOIN accounts a ON a.id = s.account_id
|
||||
LEFT JOIN transactions t ON t.source_file_id = s.id
|
||||
@@ -421,7 +428,7 @@ func (d *DB) SourceFiles() ([]SourceFileInfo, error) {
|
||||
var out []SourceFileInfo
|
||||
for rows.Next() {
|
||||
var f SourceFileInfo
|
||||
if err := rows.Scan(&f.AccountSlug, &f.Path, &f.SHA256, &f.ImportedAt, &f.Added); err != nil {
|
||||
if err := rows.Scan(&f.AccountSlug, &f.Path, &f.SHA256, &f.ImportedAt, &f.Rows, &f.Added); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, f)
|
||||
@@ -429,6 +436,55 @@ func (d *DB) SourceFiles() ([]SourceFileInfo, error) {
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ForgetSourceFile removes a statement from the index: its transactions, any
|
||||
// transfer pairing that used them, and the record of it, returning how many
|
||||
// transactions went. Pairing is derived state that the next Link rewrites, so
|
||||
// dropping a pair here loses nothing.
|
||||
//
|
||||
// A transaction two overlapping statements share is stored once, under the
|
||||
// file that brought it first, so removing that file can take rows another
|
||||
// statement still holds. The account's other statements therefore forget
|
||||
// their checksums, and the next import re-reads them and restores those rows.
|
||||
func (d *DB) ForgetSourceFile(accountSlug, path string) (int, error) {
|
||||
tx, err := d.sql.Begin()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
var id, accountID int64
|
||||
err = tx.QueryRow(`
|
||||
SELECT s.id, s.account_id FROM source_files s JOIN accounts a ON a.id = s.account_id
|
||||
WHERE a.slug = ? AND s.path = ?`, accountSlug, path).Scan(&id, &accountID)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("find source file %s: %w", path, err)
|
||||
}
|
||||
if _, err := tx.Exec(`
|
||||
DELETE FROM transfers WHERE
|
||||
out_txn_id IN (SELECT id FROM transactions WHERE source_file_id = ?) OR
|
||||
in_txn_id IN (SELECT id FROM transactions WHERE source_file_id = ?)`, id, id); err != nil {
|
||||
return 0, fmt.Errorf("forget transfers from %s: %w", path, err)
|
||||
}
|
||||
res, err := tx.Exec(`DELETE FROM transactions WHERE source_file_id = ?`, id)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("forget transactions from %s: %w", path, err)
|
||||
}
|
||||
n, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM source_files WHERE id = ?`, id); err != nil {
|
||||
return 0, fmt.Errorf("forget %s: %w", path, err)
|
||||
}
|
||||
if _, err := tx.Exec(`UPDATE source_files SET sha256 = '' WHERE account_id = ?`, accountID); err != nil {
|
||||
return 0, fmt.Errorf("mark %s's statements for re-reading: %w", accountSlug, err)
|
||||
}
|
||||
return int(n), tx.Commit()
|
||||
}
|
||||
|
||||
// Tags lists every tag in use, for completion in the rule builder.
|
||||
func (d *DB) Tags() ([]string, error) {
|
||||
rows, err := d.sql.Query(`
|
||||
|
||||
Reference in New Issue
Block a user