From f6e560510817b5c21528de0ee76d81ab4cf40d8d Mon Sep 17 00:00:00 2001 From: Nikola Petrov Date: Mon, 12 Oct 2026 20:53:27 +0200 Subject: [PATCH] Add startup.si scraper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit startup.si has no public jobs API or feed, so parse the listing page. It lists every posting since 2019 on one page, so only postings from the last 90 days are kept. Also decode common named HTML entities (š, ž, č, dashes, quotes) case-sensitively. Co-Authored-By: Claude Opus 5.5 --- scrapper/README.md | 7 +++ scrapper/src/scrapers/http.ts | 4 +- scrapper/src/scrapers/index.ts | 2 + scrapper/src/scrapers/startupsi.ts | 94 ++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 scrapper/src/scrapers/startupsi.ts diff --git a/scrapper/README.md b/scrapper/README.md index 7d901b4..92203c1 100644 --- a/scrapper/README.md +++ b/scrapper/README.md @@ -4,6 +4,13 @@ Scrapes job sites once a day, stores results in SQLite (`data/jobs.sqlite`), and No runtime dependencies. It uses Bun's built-in SQLite, HTTP server and HTMLRewriter. +## Sources + +| Source | How | Notes | +|---|---|---| +| slo-tech.com (Ljubljana) | HTML scrape | ~20 newest jobs | +| startup.si | HTML scrape (no public API or feed) | Site lists every posting since 2019 on one page, so only the last 90 days are kept (`startupSi(days)` in `src/scrapers/index.ts`) | + ## Run ```sh diff --git a/scrapper/src/scrapers/http.ts b/scrapper/src/scrapers/http.ts index 97e83d5..65a771c 100644 --- a/scrapper/src/scrapers/http.ts +++ b/scrapper/src/scrapers/http.ts @@ -29,6 +29,8 @@ export async function fetchHtml(url: string): Promise { const NAMED_ENTITIES: Record = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", raquo: "»", laquo: "«", + scaron: "š", Scaron: "Š", zcaron: "ž", Zcaron: "Ž", ccaron: "č", Ccaron: "Č", + ndash: "–", mdash: "—", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", bdquo: "„", hellip: "…", }; /** Decode HTML entities (HTMLRewriter text chunks are not decoded). */ @@ -38,7 +40,7 @@ export function decodeEntities(s: string): string { const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); return Number.isFinite(code) ? String.fromCodePoint(code) : m; } - return NAMED_ENTITIES[e.toLowerCase()] ?? m; + return NAMED_ENTITIES[e] ?? NAMED_ENTITIES[e.toLowerCase()] ?? m; }); } diff --git a/scrapper/src/scrapers/index.ts b/scrapper/src/scrapers/index.ts index e5afb1a..ba1a3a0 100644 --- a/scrapper/src/scrapers/index.ts +++ b/scrapper/src/scrapers/index.ts @@ -1,4 +1,5 @@ import { sloTech } from "./slotech"; +import { startupSi } from "./startupsi"; import type { Scraper } from "./types"; /** @@ -8,6 +9,7 @@ import type { Scraper } from "./types"; */ export const scrapers: Scraper[] = [ sloTech("Ljubljana"), + startupSi(), ]; export type { Scraper, ScrapedJob } from "./types"; diff --git a/scrapper/src/scrapers/startupsi.ts b/scrapper/src/scrapers/startupsi.ts new file mode 100644 index 0000000..b183b9b --- /dev/null +++ b/scrapper/src/scrapers/startupsi.ts @@ -0,0 +1,94 @@ +import { clean, fetchHtml } from "./http"; +import type { ScrapedJob, Scraper } from "./types"; + +// startup.si has no public jobs API or feed, so we parse the listing page. It holds every posting +// on a single page (no pagination), going back years, so old postings are filtered out by age. +const LIST_URL = "https://www.startup.si/sl-si/za-startupe/oglasi/zaposlitve"; + +const MONTHS = ["januar", "februar", "marec", "april", "maj", "junij", "julij", "avgust", "september", "oktober", "november", "december"]; + +/** "21. september 2026" -> ISO date */ +function parseSlDate(s: string): string | undefined { + const m = /(\d{1,2})\.\s*([a-zčšž]+)\s+(\d{4})/i.exec(s); + const month = m ? MONTHS.indexOf(m[2]!.toLowerCase()) : -1; + if (!m || month < 0) return undefined; + return new Date(Date.UTC(Number(m[3]), month, Number(m[1]))).toISOString(); +} + +/** Links on this site are hand-entered: some are empty, relative, or missing the scheme. */ +function normalizeUrl(href: string): string { + const h = clean(href); + if (!h) return LIST_URL; + if (/^www\./i.test(h)) return `https://${h}`; + try { + return new URL(h, LIST_URL + "/").href; + } catch { + return LIST_URL; + } +} + +/** Scraper for startup.si job ads. Only postings from the last `maxAgeDays` days are returned. */ +export function startupSi(maxAgeDays = 90): Scraper { + return { + id: "startup-si", + name: "Startup.si", + async scrape() { + const jobs = await parseStartupSi(await fetchHtml(LIST_URL)); + if (jobs.length === 0) throw new Error("no job cards found (site layout changed?)"); + const cutoff = Date.now() - maxAgeDays * 86400_000; + return jobs.filter((j) => !j.postedAt || new Date(j.postedAt).getTime() >= cutoff); + }, + }; +} + +export async function parseStartupSi(html: string): Promise { + type Draft = { id: string; title: string; date: string; href?: string }; + const drafts: Draft[] = []; + let cur: Draft | undefined; + let lastHref = ""; + + await new HTMLRewriter() + .on(".advertisment-card", { + element() { + cur = { id: "", title: "", date: "" }; + drafts.push(cur); + }, + }) + .on(".advertisment-card .right > .title", { + text(t) { + if (cur) cur.title += t.text; + }, + }) + .on(".advertisment-card .title button", { + element(el) { + if (cur) cur.id = (el.getAttribute("data-target") ?? "").replace(/^#description/, ""); + }, + }) + .on(".advertisment-card .published-info b", { + text(t) { + if (cur) cur.date += t.text; + }, + }) + // The "preberi več" (read more) link is an wrapping span.read-more; descriptions can contain other links. + .on(".advertisment-card a", { + element(el) { + lastHref = el.getAttribute("href") ?? ""; + }, + }) + .on(".advertisment-card span.read-more", { + element() { + if (cur) cur.href = lastHref; + }, + }) + .transform(new Response(html)) + .text(); + + return drafts + .filter((d) => d.id && clean(d.title)) + .map((d) => ({ + externalId: d.id, + title: clean(d.title), + url: normalizeUrl(d.href ?? ""), + postedAt: parseSlDate(clean(d.date)), + })); +}