nikolaandClaude Opus 5.5
ad2fa2b775
Optius publishes an RSS feed with region/field filters, so no HTML scraping is needed. Also move the duplicated slug helper into http.ts (output unchanged, so existing source IDs and URLs are unaffected). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
57 lines
2.4 KiB
TypeScript
57 lines
2.4 KiB
TypeScript
// Shared helpers for scrapers.
|
||
|
||
export const USER_AGENT =
|
||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36";
|
||
|
||
// Upper half (0xA0-0xFF) of ISO-8859-2, which Bun's TextDecoder does not support.
|
||
const LATIN2_HIGH =
|
||
" Ą˘Ł¤ĽŚ§¨ŠŞŤŹŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
|
||
|
||
function decodeLatin2(bytes: Uint8Array): string {
|
||
let out = "";
|
||
for (const b of bytes) out += b < 0xa0 ? String.fromCharCode(b) : LATIN2_HIGH[b - 0xa0];
|
||
return out;
|
||
}
|
||
|
||
/** Fetch a page and decode it according to its declared charset. */
|
||
export async function fetchHtml(url: string): Promise<string> {
|
||
const res = await fetch(url, { headers: { "User-Agent": USER_AGENT } });
|
||
if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`);
|
||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||
const charset = /charset=([\w-]+)/i.exec(res.headers.get("content-type") ?? "")?.[1]?.toLowerCase() ?? "utf-8";
|
||
if (charset === "iso-8859-2" || charset === "latin2") return decodeLatin2(bytes);
|
||
try {
|
||
return new TextDecoder(charset).decode(bytes);
|
||
} catch {
|
||
return new TextDecoder("utf-8").decode(bytes);
|
||
}
|
||
}
|
||
|
||
const NAMED_ENTITIES: Record<string, string> = {
|
||
amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", raquo: "»", laquo: "«",
|
||
scaron: "š", Scaron: "Š", zcaron: "ž", Zcaron: "Ž", ccaron: "č", Ccaron: "Č",
|
||
ndash: "–", mdash: "—", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", bdquo: "„", hellip: "…",
|
||
};
|
||
|
||
/** Decode HTML entities (HTMLRewriter text chunks are not decoded). */
|
||
export function decodeEntities(s: string): string {
|
||
return s.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (m, e: string) => {
|
||
if (e[0] === "#") {
|
||
const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
|
||
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
|
||
}
|
||
return NAMED_ENTITIES[e] ?? NAMED_ENTITIES[e.toLowerCase()] ?? m;
|
||
});
|
||
}
|
||
|
||
export const clean = (s: string) => decodeEntities(s).replace(/\s+/g, " ").trim();
|
||
|
||
/** ASCII URL/ID slug: "Inženir (m/ž)" -> "inzenir-m-z". */
|
||
export const slugify = (s: string) =>
|
||
s
|
||
.normalize("NFD")
|
||
.replace(/[̀-ͯ]/g, "")
|
||
.replace(/[^A-Za-z0-9]+/g, "-")
|
||
.replace(/^-|-$/g, "")
|
||
.toLowerCase();
|