Add job board scraper: modular scrapers, SQLite storage, web UI
Scrapes slo-tech.com (Ljubljana) every 24h, stores jobs in SQLite and tracks which are new vs. seen. New sites can be added as modules in src/scrapers/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
// Shared helpers for scrapers.
|
||||
|
||||
const USER_AGENT =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36";
|
||||
|
||||
// Upper half (0xA0-0xFF) of ISO-8859-2, which Bun's TextDecoder does not support.
|
||||
const LATIN2_HIGH =
|
||||
" Ą˘Ł¤ĽŚ§¨ŠŞŤŹŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
|
||||
|
||||
function decodeLatin2(bytes: Uint8Array): string {
|
||||
let out = "";
|
||||
for (const b of bytes) out += b < 0xa0 ? String.fromCharCode(b) : LATIN2_HIGH[b - 0xa0];
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Fetch a page and decode it according to its declared charset. */
|
||||
export async function fetchHtml(url: string): Promise<string> {
|
||||
const res = await fetch(url, { headers: { "User-Agent": USER_AGENT } });
|
||||
if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`);
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
const charset = /charset=([\w-]+)/i.exec(res.headers.get("content-type") ?? "")?.[1]?.toLowerCase() ?? "utf-8";
|
||||
if (charset === "iso-8859-2" || charset === "latin2") return decodeLatin2(bytes);
|
||||
try {
|
||||
return new TextDecoder(charset).decode(bytes);
|
||||
} catch {
|
||||
return new TextDecoder("utf-8").decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
const NAMED_ENTITIES: Record<string, string> = {
|
||||
amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", raquo: "»", laquo: "«",
|
||||
};
|
||||
|
||||
/** Decode HTML entities (HTMLRewriter text chunks are not decoded). */
|
||||
export function decodeEntities(s: string): string {
|
||||
return s.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (m, e: string) => {
|
||||
if (e[0] === "#") {
|
||||
const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
|
||||
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return NAMED_ENTITIES[e.toLowerCase()] ?? m;
|
||||
});
|
||||
}
|
||||
|
||||
export const clean = (s: string) => decodeEntities(s).replace(/\s+/g, " ").trim();
|
||||
Reference in New Issue
Block a user