Name NLB uploads, delete statements, and stop importing on upload

Four changes to statement handling in the web app, made together and
touching the same upload and statements-list code.

Name NLB uploads by statement date. parser.Namer is an optional
interface, like Warner, through which a parser names its statements;
nlb reads the "Datum izpiska" from the izpisek header and names it
izpisek_YYYY_MM_DD, lowercase, extension included -- ported from the
rename_izpiski.py it replaces. Uploads are staged as dotfiles, invisible
to import, so the parser can read them; two downloads of one statement
then meet under one name and the second is recognised as already there,
while a different statement of the same date is numbered _2 as the
script did. Only uploads are named: source_files records statements by
path, so renaming a file already in a folder would orphan its rows.

Delete a statement from the statements list. The file is removed from
disk for good -- the page says so before it asks -- and
store.ForgetSourceFile drops its transactions and their transfer rows.
A row two overlapping statements share is stored once, under the file
imported first, so it goes too; the account's other statements forget
their checksums and show as changed until the next Import re-reads them
and restores it. A file already gone from disk can be forgotten.

Upload and delete no longer import. Importing stays the user's call,
made with the Import button, so a batch can be put together and looked
over first. Delete still re-pairs transfers, which reads no statement.

Show rows and new rows per statement. The list read "0" for a file
whose rows an earlier, overlapping statement already held, which looked
like a file that failed to parse. source_files now records how many
transactions each statement holds, and the list reads "3 rows · 0 new".

This adds a column the code reads, so an index built by an earlier
version fails with "no such column: s.rows": delete index.db and import
again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-02 21:48:29 +02:00
co-authored by Claude Opus 5.5
parent 9ad6c05f9a
commit a5f7541980
12 changed files with 652 additions and 146 deletions
+64 -8
View File
@@ -37,6 +37,9 @@ CREATE TABLE IF NOT EXISTS source_files (
path TEXT NOT NULL,
sha256 TEXT NOT NULL,
imported_at TEXT NOT NULL,
-- How many transactions the statement holds, as parsed. Not how many it
-- brought in: rows an overlapping statement already had are deduplicated.
rows INTEGER NOT NULL DEFAULT 0,
UNIQUE(account_id, path)
);
@@ -130,15 +133,17 @@ func (d *DB) Accounts() ([]model.Account, error) {
return out, rows.Err()
}
// SourceFile records that a statement file was imported, returning its id.
func (d *DB) SourceFile(accountID int64, path, sha, importedAt string) (int64, error) {
// SourceFile records that a statement file was imported, holding rows
// transactions, and returns its id.
func (d *DB) SourceFile(accountID int64, path, sha, importedAt string, rows int) (int64, error) {
_, err := d.sql.Exec(`
INSERT INTO source_files (account_id, path, sha256, imported_at)
VALUES (?, ?, ?, ?)
INSERT INTO source_files (account_id, path, sha256, imported_at, rows)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(account_id, path) DO UPDATE SET
sha256 = excluded.sha256,
imported_at = excluded.imported_at`,
accountID, path, sha, importedAt)
imported_at = excluded.imported_at,
rows = excluded.rows`,
accountID, path, sha, importedAt, rows)
if err != nil {
return 0, fmt.Errorf("record source file %s: %w", path, err)
}
@@ -399,6 +404,8 @@ type SourceFileInfo struct {
Path string // relative to the data root, as the importer records it
SHA256 string
ImportedAt string
// Rows is how many transactions the statement holds, as parsed.
Rows int
// Added counts the transactions this file introduced. A row seen again in
// an overlapping statement is deduplicated and stays with the file that
// brought it first, so this is not how many rows the file holds.
@@ -408,7 +415,7 @@ type SourceFileInfo struct {
// SourceFiles lists every statement the index has imported.
func (d *DB) SourceFiles() ([]SourceFileInfo, error) {
rows, err := d.sql.Query(`
SELECT a.slug, s.path, s.sha256, s.imported_at, COUNT(t.id)
SELECT a.slug, s.path, s.sha256, s.imported_at, s.rows, COUNT(t.id)
FROM source_files s
JOIN accounts a ON a.id = s.account_id
LEFT JOIN transactions t ON t.source_file_id = s.id
@@ -421,7 +428,7 @@ func (d *DB) SourceFiles() ([]SourceFileInfo, error) {
var out []SourceFileInfo
for rows.Next() {
var f SourceFileInfo
if err := rows.Scan(&f.AccountSlug, &f.Path, &f.SHA256, &f.ImportedAt, &f.Added); err != nil {
if err := rows.Scan(&f.AccountSlug, &f.Path, &f.SHA256, &f.ImportedAt, &f.Rows, &f.Added); err != nil {
return nil, err
}
out = append(out, f)
@@ -429,6 +436,55 @@ func (d *DB) SourceFiles() ([]SourceFileInfo, error) {
return out, rows.Err()
}
// ForgetSourceFile removes a statement from the index: its transactions, any
// transfer pairing that used them, and the record of it, returning how many
// transactions went. Pairing is derived state that the next Link rewrites, so
// dropping a pair here loses nothing.
//
// A transaction two overlapping statements share is stored once, under the
// file that brought it first, so removing that file can take rows another
// statement still holds. The account's other statements therefore forget
// their checksums, and the next import re-reads them and restores those rows.
func (d *DB) ForgetSourceFile(accountSlug, path string) (int, error) {
tx, err := d.sql.Begin()
if err != nil {
return 0, err
}
defer tx.Rollback()
var id, accountID int64
err = tx.QueryRow(`
SELECT s.id, s.account_id FROM source_files s JOIN accounts a ON a.id = s.account_id
WHERE a.slug = ? AND s.path = ?`, accountSlug, path).Scan(&id, &accountID)
if err == sql.ErrNoRows {
return 0, nil
}
if err != nil {
return 0, fmt.Errorf("find source file %s: %w", path, err)
}
if _, err := tx.Exec(`
DELETE FROM transfers WHERE
out_txn_id IN (SELECT id FROM transactions WHERE source_file_id = ?) OR
in_txn_id IN (SELECT id FROM transactions WHERE source_file_id = ?)`, id, id); err != nil {
return 0, fmt.Errorf("forget transfers from %s: %w", path, err)
}
res, err := tx.Exec(`DELETE FROM transactions WHERE source_file_id = ?`, id)
if err != nil {
return 0, fmt.Errorf("forget transactions from %s: %w", path, err)
}
n, err := res.RowsAffected()
if err != nil {
return 0, err
}
if _, err := tx.Exec(`DELETE FROM source_files WHERE id = ?`, id); err != nil {
return 0, fmt.Errorf("forget %s: %w", path, err)
}
if _, err := tx.Exec(`UPDATE source_files SET sha256 = '' WHERE account_id = ?`, accountID); err != nil {
return 0, fmt.Errorf("mark %s's statements for re-reading: %w", accountSlug, err)
}
return int(n), tx.Commit()
}
// Tags lists every tag in use, for completion in the rule builder.
func (d *DB) Tags() ([]string, error) {
rows, err := d.sql.Query(`