Add job board scraper: modular scrapers, SQLite storage, web UI

Scrapes slo-tech.com (Ljubljana) every 24h, stores jobs in SQLite and
tracks which are new vs. seen. New sites can be added as modules in
src/scrapers/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-12 20:45:40 +02:00
co-authored by Claude Opus 5.5
parent 42f97ea073
commit 0324ff7f2a
12 changed files with 697 additions and 0 deletions
+45
View File
@@ -0,0 +1,45 @@
// Shared helpers for scrapers.
const USER_AGENT =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36";
// Upper half (0xA0-0xFF) of ISO-8859-2, which Bun's TextDecoder does not support.
const LATIN2_HIGH =
" Ą˘Ł¤ĽŚ§¨ŠŞŤŹ­ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
function decodeLatin2(bytes: Uint8Array): string {
let out = "";
for (const b of bytes) out += b < 0xa0 ? String.fromCharCode(b) : LATIN2_HIGH[b - 0xa0];
return out;
}
/** Fetch a page and decode it according to its declared charset. */
export async function fetchHtml(url: string): Promise<string> {
const res = await fetch(url, { headers: { "User-Agent": USER_AGENT } });
if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`);
const bytes = new Uint8Array(await res.arrayBuffer());
const charset = /charset=([\w-]+)/i.exec(res.headers.get("content-type") ?? "")?.[1]?.toLowerCase() ?? "utf-8";
if (charset === "iso-8859-2" || charset === "latin2") return decodeLatin2(bytes);
try {
return new TextDecoder(charset).decode(bytes);
} catch {
return new TextDecoder("utf-8").decode(bytes);
}
}
const NAMED_ENTITIES: Record<string, string> = {
amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", raquo: "»", laquo: "«",
};
/** Decode HTML entities (HTMLRewriter text chunks are not decoded). */
export function decodeEntities(s: string): string {
return s.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (m, e: string) => {
if (e[0] === "#") {
const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
}
return NAMED_ENTITIES[e.toLowerCase()] ?? m;
});
}
export const clean = (s: string) => decodeEntities(s).replace(/\s+/g, " ").trim();