nikolaandClaude Opus 5.5
f6e5605108
startup.si has no public jobs API or feed, so parse the listing page. It lists every posting since 2019 on one page, so only postings from the last 90 days are kept. Also decode common named HTML entities (š, ž, č, dashes, quotes) case-sensitively. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
48 lines
2.1 KiB
TypeScript
48 lines
2.1 KiB
TypeScript
// Shared helpers for scrapers.
|
||
|
||
const USER_AGENT =
|
||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36";
|
||
|
||
// Upper half (0xA0-0xFF) of ISO-8859-2, which Bun's TextDecoder does not support.
|
||
const LATIN2_HIGH =
|
||
" Ą˘Ł¤ĽŚ§¨ŠŞŤŹŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
|
||
|
||
function decodeLatin2(bytes: Uint8Array): string {
|
||
let out = "";
|
||
for (const b of bytes) out += b < 0xa0 ? String.fromCharCode(b) : LATIN2_HIGH[b - 0xa0];
|
||
return out;
|
||
}
|
||
|
||
/** Fetch a page and decode it according to its declared charset. */
|
||
export async function fetchHtml(url: string): Promise<string> {
|
||
const res = await fetch(url, { headers: { "User-Agent": USER_AGENT } });
|
||
if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`);
|
||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||
const charset = /charset=([\w-]+)/i.exec(res.headers.get("content-type") ?? "")?.[1]?.toLowerCase() ?? "utf-8";
|
||
if (charset === "iso-8859-2" || charset === "latin2") return decodeLatin2(bytes);
|
||
try {
|
||
return new TextDecoder(charset).decode(bytes);
|
||
} catch {
|
||
return new TextDecoder("utf-8").decode(bytes);
|
||
}
|
||
}
|
||
|
||
const NAMED_ENTITIES: Record<string, string> = {
|
||
amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", raquo: "»", laquo: "«",
|
||
scaron: "š", Scaron: "Š", zcaron: "ž", Zcaron: "Ž", ccaron: "č", Ccaron: "Č",
|
||
ndash: "–", mdash: "—", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", bdquo: "„", hellip: "…",
|
||
};
|
||
|
||
/** Decode HTML entities (HTMLRewriter text chunks are not decoded). */
|
||
export function decodeEntities(s: string): string {
|
||
return s.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (m, e: string) => {
|
||
if (e[0] === "#") {
|
||
const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
|
||
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
|
||
}
|
||
return NAMED_ENTITIES[e] ?? NAMED_ENTITIES[e.toLowerCase()] ?? m;
|
||
});
|
||
}
|
||
|
||
export const clean = (s: string) => decodeEntities(s).replace(/\s+/g, " ").trim();
|