diff --git a/scrapper/README.md b/scrapper/README.md index 7d901b4..92203c1 100644 --- a/scrapper/README.md +++ b/scrapper/README.md @@ -4,6 +4,13 @@ Scrapes job sites once a day, stores results in SQLite (`data/jobs.sqlite`), and No runtime dependencies. It uses Bun's built-in SQLite, HTTP server and HTMLRewriter. +## Sources + +| Source | How | Notes | +|---|---|---| +| slo-tech.com (Ljubljana) | HTML scrape | ~20 newest jobs | +| startup.si | HTML scrape (no public API or feed) | Site lists every posting since 2019 on one page, so only the last 90 days are kept (`startupSi(days)` in `src/scrapers/index.ts`) | + ## Run ```sh diff --git a/scrapper/src/scrapers/http.ts b/scrapper/src/scrapers/http.ts index 97e83d5..65a771c 100644 --- a/scrapper/src/scrapers/http.ts +++ b/scrapper/src/scrapers/http.ts @@ -29,6 +29,8 @@ export async function fetchHtml(url: string): Promise { const NAMED_ENTITIES: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", raquo: "»", laquo: "«", + scaron: "š", Scaron: "Š", zcaron: "ž", Zcaron: "Ž", ccaron: "č", Ccaron: "Č", + ndash: "–", mdash: "—", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", bdquo: "„", hellip: "…", }; /** Decode HTML entities (HTMLRewriter text chunks are not decoded). */ @@ -38,7 +40,7 @@ export function decodeEntities(s: string): string { const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); return Number.isFinite(code) ? String.fromCodePoint(code) : m; } - return NAMED_ENTITIES[e.toLowerCase()] ?? m; + return NAMED_ENTITIES[e] ?? NAMED_ENTITIES[e.toLowerCase()] ?? m; }); } diff --git a/scrapper/src/scrapers/index.ts b/scrapper/src/scrapers/index.ts index e5afb1a..ba1a3a0 100644 --- a/scrapper/src/scrapers/index.ts +++ b/scrapper/src/scrapers/index.ts @@ -1,4 +1,5 @@ import { sloTech } from "./slotech"; +import { startupSi } from "./startupsi"; import type { Scraper } from "./types"; /** @@ -8,6 +9,7 @@ import type { Scraper } from "./types"; */ export const scrapers: Scraper[] = [ sloTech("Ljubljana"), + startupSi(), ]; export type { Scraper, ScrapedJob } from "./types"; diff --git a/scrapper/src/scrapers/startupsi.ts b/scrapper/src/scrapers/startupsi.ts new file mode 100644 index 0000000..b183b9b --- /dev/null +++ b/scrapper/src/scrapers/startupsi.ts @@ -0,0 +1,94 @@ +import { clean, fetchHtml } from "./http"; +import type { ScrapedJob, Scraper } from "./types"; + +// startup.si has no public jobs API or feed, so we parse the listing page. It holds every posting +// on a single page (no pagination), going back years, so old postings are filtered out by age. +const LIST_URL = "https://www.startup.si/sl-si/za-startupe/oglasi/zaposlitve"; + +const MONTHS = ["januar", "februar", "marec", "april", "maj", "junij", "julij", "avgust", "september", "oktober", "november", "december"]; + +/** "21. september 2026" -> ISO date */ +function parseSlDate(s: string): string | undefined { + const m = /(\d{1,2})\.\s*([a-zčšž]+)\s+(\d{4})/i.exec(s); + const month = m ? MONTHS.indexOf(m[2]!.toLowerCase()) : -1; + if (!m || month < 0) return undefined; + return new Date(Date.UTC(Number(m[3]), month, Number(m[1]))).toISOString(); +} + +/** Links on this site are hand-entered: some are empty, relative, or missing the scheme. */ +function normalizeUrl(href: string): string { + const h = clean(href); + if (!h) return LIST_URL; + if (/^www\./i.test(h)) return `https://${h}`; + try { + return new URL(h, LIST_URL + "/").href; + } catch { + return LIST_URL; + } +} + +/** Scraper for startup.si job ads. Only postings from the last `maxAgeDays` days are returned. */ +export function startupSi(maxAgeDays = 90): Scraper { + return { + id: "startup-si", + name: "Startup.si", + async scrape() { + const jobs = await parseStartupSi(await fetchHtml(LIST_URL)); + if (jobs.length === 0) throw new Error("no job cards found (site layout changed?)"); + const cutoff = Date.now() - maxAgeDays * 86400_000; + return jobs.filter((j) => !j.postedAt || new Date(j.postedAt).getTime() >= cutoff); + }, + }; +} + +export async function parseStartupSi(html: string): Promise { + type Draft = { id: string; title: string; date: string; href?: string }; + const drafts: Draft[] = []; + let cur: Draft | undefined; + let lastHref = ""; + + await new HTMLRewriter() + .on(".advertisment-card", { + element() { + cur = { id: "", title: "", date: "" }; + drafts.push(cur); + }, + }) + .on(".advertisment-card .right > .title", { + text(t) { + if (cur) cur.title += t.text; + }, + }) + .on(".advertisment-card .title button", { + element(el) { + if (cur) cur.id = (el.getAttribute("data-target") ?? "").replace(/^#description/, ""); + }, + }) + .on(".advertisment-card .published-info b", { + text(t) { + if (cur) cur.date += t.text; + }, + }) + // The "preberi več" (read more) link is an wrapping span.read-more; descriptions can contain other links. + .on(".advertisment-card a", { + element(el) { + lastHref = el.getAttribute("href") ?? ""; + }, + }) + .on(".advertisment-card span.read-more", { + element() { + if (cur) cur.href = lastHref; + }, + }) + .transform(new Response(html)) + .text(); + + return drafts + .filter((d) => d.id && clean(d.title)) + .map((d) => ({ + externalId: d.id, + title: clean(d.title), + url: normalizeUrl(d.href ?? ""), + postedAt: parseSlDate(clean(d.date)), + })); +}