diff --git a/scrapper/.gitignore b/scrapper/.gitignore new file mode 100644 index 0000000..ad250eb --- /dev/null +++ b/scrapper/.gitignore @@ -0,0 +1,2 @@ +node_modules +data/ diff --git a/scrapper/README.md b/scrapper/README.md new file mode 100644 index 0000000..7d901b4 --- /dev/null +++ b/scrapper/README.md @@ -0,0 +1,41 @@ +# Job Board + +Scrapes job sites once a day, stores results in SQLite (`data/jobs.sqlite`), and shows them in a web UI where you can mark jobs as seen. + +No runtime dependencies. It uses Bun's built-in SQLite, HTTP server and HTMLRewriter. + +## Run + +```sh +bun install +bun start # http://localhost:3000 +``` + +The server scrapes every 24h. On startup it waits out the rest of the 24h since the last scrape (or scrapes right away if none happened in the last 24h). +You can also click **Scrape now** in the UI, or run `bun run scrape` once (e.g. from cron). + +Env vars: `PORT` (3000), `SCRAPE_INTERVAL_HOURS` (24), `DB_PATH` (`data/jobs.sqlite`). + +## UI + +- **New / Seen / All** tabs. Opening a job, or clicking *Mark seen*, moves it to Seen. +- **no longer listed** means the job was missing from that source's latest successful scrape. +- Search filters on title, company and tags. Recent scrape runs (and errors) are listed at the bottom. + +## Adding a site + +1. Create `src/scrapers/.ts` exporting an object that implements `Scraper` (`src/scrapers/types.ts`): + ```ts + export const mySite: Scraper = { + id: "my-site", // stable, stored in DB + name: "My Site", + async scrape() { + const html = await fetchHtml("https://..."); // helper in ./http.ts + return [/* { externalId, title, url, company?, location?, tags?, postedAt? } */]; + }, + }; + ``` +2. Add it to the list in `src/scrapers/index.ts`. + +`externalId` must stay stable for a job, because it is what decides whether a job is new. +For another slo-tech city, just add `sloTech("Maribor")` to the list. diff --git a/scrapper/bun.lock b/scrapper/bun.lock new file mode 100644 index 0000000..ed3b17d --- /dev/null +++ b/scrapper/bun.lock @@ -0,0 +1,23 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "job-board", + "devDependencies": { + "@types/bun": "latest", + }, + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.4.3", "", { "dependencies": { "bun-types": "1.4.3" } }, "sha512-N67Pqy8JXbAY/j0tNraS3V5mubGvc6rrrNDmDCtZLlFzccbUJPDb0tvgUBC5q4m8ybkP93Jcn1QB+RYQ6PvBhw=="], + + "@types/node": ["@types/node@26.6.5", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg=="], + + "bun-types": ["bun-types@1.4.3", "", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-tinRJNGOmUN+tEAsn2y2e942n9JHRHR42iUYzDH75NmhJqVBokGSQkkfSFdNWwsANIdrK6ACH9t01M452YyTwg=="], + + "undici-types": ["undici-types@8.11.2", "", {}, "sha512-iMVNmWZ0leK/goS6eXMizSzmm9CDWtyphwbaCms3DNLqRxDL+mMoNVcZMTyyVgXP0N+Z8neAMzDoUOUJL8veKg=="], + + "@types/node/undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + } +} diff --git a/scrapper/package.json b/scrapper/package.json new file mode 100644 index 0000000..9fc5429 --- /dev/null +++ b/scrapper/package.json @@ -0,0 +1,13 @@ +{ + "name": "job-board", + "private": true, + "type": "module", + "scripts": { + "start": "bun run src/server.ts", + "dev": "bun --watch run src/server.ts", + "scrape": "bun run src/scrape.ts" + }, + "devDependencies": { + "@types/bun": "latest" + } +} diff --git a/scrapper/src/db.ts b/scrapper/src/db.ts new file mode 100644 index 0000000..96a887b --- /dev/null +++ b/scrapper/src/db.ts @@ -0,0 +1,176 @@ +import { Database } from "bun:sqlite"; +import { mkdirSync } from "node:fs"; +import { dirname } from "node:path"; +import type { ScrapedJob } from "./scrapers/types"; + +const DB_PATH = process.env.DB_PATH ?? `${import.meta.dir}/../data/jobs.sqlite`; +mkdirSync(dirname(DB_PATH), { recursive: true }); + +export const db = new Database(DB_PATH, { create: true, strict: true }); +db.run("PRAGMA journal_mode = WAL"); +db.run(` + CREATE TABLE IF NOT EXISTS jobs ( + id INTEGER PRIMARY KEY, + source TEXT NOT NULL, + external_id TEXT NOT NULL, + title TEXT NOT NULL, + company TEXT, + url TEXT NOT NULL, + location TEXT, + tags TEXT NOT NULL DEFAULT '[]', + posted_at TEXT, + first_seen_at TEXT NOT NULL, -- when the scraper first found it + last_seen_at TEXT NOT NULL, -- last scrape that still listed it + seen_at TEXT, -- when you marked it as seen (NULL = new) + UNIQUE (source, external_id) + ) +`); +db.run(` + CREATE TABLE IF NOT EXISTS scrape_runs ( + id INTEGER PRIMARY KEY, + source TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT, + found INTEGER, + new_count INTEGER, + error TEXT + ) +`); + +export interface JobRow { + id: number; + source: string; + external_id: string; + title: string; + company: string | null; + url: string; + location: string | null; + tags: string; + posted_at: string | null; + first_seen_at: string; + last_seen_at: string; + seen_at: string | null; + /** 0 if the job was missing from the source's latest successful scrape. */ + still_listed: number; +} + +export interface RunRow { + id: number; + source: string; + started_at: string; + finished_at: string | null; + found: number | null; + new_count: number | null; + error: string | null; +} + +const upsertJob = db.prepare(` + INSERT INTO jobs (source, external_id, title, company, url, location, tags, posted_at, first_seen_at, last_seen_at) + VALUES ($source, $externalId, $title, $company, $url, $location, $tags, $postedAt, $now, $now) + ON CONFLICT (source, external_id) DO UPDATE SET + title = excluded.title, company = excluded.company, url = excluded.url, location = excluded.location, + tags = excluded.tags, posted_at = excluded.posted_at, last_seen_at = excluded.last_seen_at + RETURNING first_seen_at = $now AS is_new +`); + +/** Insert or refresh scraped jobs; returns how many were not in the DB before. */ +export const saveJobs = db.transaction((source: string, jobs: ScrapedJob[], now: string): number => { + let newCount = 0; + for (const j of jobs) { + const row = upsertJob.get({ + source, + externalId: j.externalId, + title: j.title, + company: j.company ?? null, + url: j.url, + location: j.location ?? null, + tags: JSON.stringify(j.tags ?? []), + postedAt: j.postedAt ?? null, + now, + }) as { is_new: number }; + newCount += row.is_new; + } + return newCount; +}); + +export function startRun(source: string, now: string): number { + const row = db + .query("INSERT INTO scrape_runs (source, started_at) VALUES (?, ?) RETURNING id") + .get(source, now) as { id: number }; + return row.id; +} + +export function finishRun(id: number, result: { found?: number; newCount?: number; error?: string }) { + db.query("UPDATE scrape_runs SET finished_at = ?, found = ?, new_count = ?, error = ? WHERE id = ?").run( + new Date().toISOString(), + result.found ?? null, + result.newCount ?? null, + result.error ?? null, + id, + ); +} + +export type Filter = "new" | "seen" | "all"; + +export function listJobs(opts: { filter: Filter; source?: string; q?: string }): JobRow[] { + const where: string[] = []; + const params: Record = {}; + if (opts.filter === "new") where.push("j.seen_at IS NULL"); + if (opts.filter === "seen") where.push("j.seen_at IS NOT NULL"); + if (opts.source) { + where.push("j.source = $source"); + params.source = opts.source; + } + if (opts.q) { + where.push("(j.title LIKE $q OR j.company LIKE $q OR j.tags LIKE $q)"); + params.q = `%${opts.q}%`; + } + return db + .query( + `SELECT j.*, + j.last_seen_at >= COALESCE( + (SELECT MAX(r.started_at) FROM scrape_runs r WHERE r.source = j.source AND r.error IS NULL AND r.finished_at IS NOT NULL), + j.last_seen_at + ) AS still_listed + FROM jobs j + ${where.length ? "WHERE " + where.join(" AND ") : ""} + ORDER BY still_listed DESC, COALESCE(j.posted_at, j.first_seen_at) DESC`, + ) + .all(params) as JobRow[]; +} + +export function counts(): { new: number; seen: number; all: number } { + return db + .query( + `SELECT SUM(seen_at IS NULL) AS new, SUM(seen_at IS NOT NULL) AS seen, COUNT(*) AS "all" FROM jobs`, + ) + .get() as { new: number; seen: number; all: number }; +} + +export function setSeen(id: number, seen: boolean) { + db.query("UPDATE jobs SET seen_at = ? WHERE id = ?").run(seen ? new Date().toISOString() : null, id); +} + +export function markAllSeen(ids: number[]) { + const stmt = db.query("UPDATE jobs SET seen_at = ? WHERE id = ? AND seen_at IS NULL"); + const now = new Date().toISOString(); + db.transaction(() => ids.forEach((id) => stmt.run(now, id)))(); +} + +export function sources(): string[] { + return (db.query("SELECT DISTINCT source FROM jobs ORDER BY source").all() as { source: string }[]).map( + (r) => r.source, + ); +} + +export function recentRuns(limit = 10): RunRow[] { + return db.query("SELECT * FROM scrape_runs ORDER BY id DESC LIMIT ?").all(limit) as RunRow[]; +} + +/** Start time of the most recent run that finished (successfully or not), across all sources. */ +export function lastRunAt(): string | null { + const row = db.query("SELECT MAX(started_at) AS t FROM scrape_runs WHERE finished_at IS NOT NULL").get() as { + t: string | null; + }; + return row.t; +} diff --git a/scrapper/src/scrape.ts b/scrapper/src/scrape.ts new file mode 100644 index 0000000..63c796f --- /dev/null +++ b/scrapper/src/scrape.ts @@ -0,0 +1,44 @@ +import { finishRun, saveJobs, startRun } from "./db"; +import { scrapers } from "./scrapers"; + +export interface RunSummary { + source: string; + found?: number; + newCount?: number; + error?: string; +} + +let running: Promise | null = null; + +/** Run every registered scraper. A failing scraper is logged and doesn't stop the others. */ +export function runAllScrapers(): Promise { + running ??= (async () => { + const results: RunSummary[] = []; + for (const s of scrapers) { + const now = new Date().toISOString(); + const runId = startRun(s.id, now); + try { + const jobs = await s.scrape(); + if (jobs.length === 0) throw new Error("scraper returned 0 jobs (site layout changed?)"); + const newCount = saveJobs(s.id, jobs, now); + finishRun(runId, { found: jobs.length, newCount }); + results.push({ source: s.id, found: jobs.length, newCount }); + console.log(`[scrape] ${s.id}: ${jobs.length} found, ${newCount} new`); + } catch (e) { + const error = e instanceof Error ? e.message : String(e); + finishRun(runId, { error }); + results.push({ source: s.id, error }); + console.error(`[scrape] ${s.id} failed: ${error}`); + } + } + return results; + })().finally(() => (running = null)); + return running; +} + +export const isScraping = () => running !== null; + +if (import.meta.main) { + const results = await runAllScrapers(); + process.exit(results.some((r) => r.error) ? 1 : 0); +} diff --git a/scrapper/src/scrapers/http.ts b/scrapper/src/scrapers/http.ts new file mode 100644 index 0000000..97e83d5 --- /dev/null +++ b/scrapper/src/scrapers/http.ts @@ -0,0 +1,45 @@ +// Shared helpers for scrapers. + +const USER_AGENT = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36"; + +// Upper half (0xA0-0xFF) of ISO-8859-2, which Bun's TextDecoder does not support. +const LATIN2_HIGH = + " Ą˘Ł¤ĽŚ§¨ŠŞŤŹ­ŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙"; + +function decodeLatin2(bytes: Uint8Array): string { + let out = ""; + for (const b of bytes) out += b < 0xa0 ? String.fromCharCode(b) : LATIN2_HIGH[b - 0xa0]; + return out; +} + +/** Fetch a page and decode it according to its declared charset. */ +export async function fetchHtml(url: string): Promise { + const res = await fetch(url, { headers: { "User-Agent": USER_AGENT } }); + if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`); + const bytes = new Uint8Array(await res.arrayBuffer()); + const charset = /charset=([\w-]+)/i.exec(res.headers.get("content-type") ?? "")?.[1]?.toLowerCase() ?? "utf-8"; + if (charset === "iso-8859-2" || charset === "latin2") return decodeLatin2(bytes); + try { + return new TextDecoder(charset).decode(bytes); + } catch { + return new TextDecoder("utf-8").decode(bytes); + } +} + +const NAMED_ENTITIES: Record = { + amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", raquo: "»", laquo: "«", +}; + +/** Decode HTML entities (HTMLRewriter text chunks are not decoded). */ +export function decodeEntities(s: string): string { + return s.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (m, e: string) => { + if (e[0] === "#") { + const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); + return Number.isFinite(code) ? String.fromCodePoint(code) : m; + } + return NAMED_ENTITIES[e.toLowerCase()] ?? m; + }); +} + +export const clean = (s: string) => decodeEntities(s).replace(/\s+/g, " ").trim(); diff --git a/scrapper/src/scrapers/index.ts b/scrapper/src/scrapers/index.ts new file mode 100644 index 0000000..e5afb1a --- /dev/null +++ b/scrapper/src/scrapers/index.ts @@ -0,0 +1,13 @@ +import { sloTech } from "./slotech"; +import type { Scraper } from "./types"; + +/** + * All active scrapers. To add a site: + * 1. create src/scrapers/.ts exporting an object implementing `Scraper` + * 2. add it to this list + */ +export const scrapers: Scraper[] = [ + sloTech("Ljubljana"), +]; + +export type { Scraper, ScrapedJob } from "./types"; diff --git a/scrapper/src/scrapers/slotech.ts b/scrapper/src/scrapers/slotech.ts new file mode 100644 index 0000000..2037eef --- /dev/null +++ b/scrapper/src/scrapers/slotech.ts @@ -0,0 +1,74 @@ +import { clean, fetchHtml } from "./http"; +import type { ScrapedJob, Scraper } from "./types"; + +const BASE = "https://slo-tech.com"; + +/** Scraper for slo-tech.com job listings, filtered by city (e.g. "Ljubljana", "Maribor"). */ +export function sloTech(city: string): Scraper { + return { + id: `slo-tech-${city.toLowerCase()}`, + name: `Slo-Tech (${city})`, + async scrape() { + const html = await fetchHtml(`${BASE}/delo/mesto/${encodeURIComponent(city)}`); + return parseSloTech(html, city); + }, + }; +} + +export async function parseSloTech(html: string, city: string): Promise { + type Draft = { href: string; title: string; company: string; tags: string[]; postedAt?: string }; + const drafts: Draft[] = []; + let cur: Draft | undefined; + + await new HTMLRewriter() + .on("table.forums tbody tr", { + element() { + cur = { href: "", title: "", company: "", tags: [] }; + drafts.push(cur); + }, + }) + .on("table.forums tbody td.name h3 a", { + element(el) { + if (cur) cur.href = el.getAttribute("href") ?? ""; + }, + text(t) { + if (cur) cur.title += t.text; + }, + }) + .on("table.forums tbody td.name span.oddelek a", { + element() { + cur?.tags.push(""); + }, + text(t) { + if (cur?.tags.length) cur.tags[cur.tags.length - 1] += t.text; + }, + }) + .on("table.forums tbody td.company", { + text(t) { + if (cur) cur.company += t.text; + }, + }) + .on("table.forums tbody td.last_msg time", { + element(el) { + if (cur) cur.postedAt = el.getAttribute("datetime") ?? undefined; + }, + }) + .transform(new Response(html)) + .text(); + + return drafts + .map((d): ScrapedJob | null => { + const id = /\/delo\/(\d+)/.exec(d.href)?.[1]; + if (!id) return null; + return { + externalId: id, + title: clean(d.title), + url: new URL(d.href, BASE).href, + company: clean(d.company) || undefined, + location: city, + tags: d.tags.map(clean).filter(Boolean), + postedAt: d.postedAt ? new Date(d.postedAt).toISOString() : undefined, + }; + }) + .filter((j): j is ScrapedJob => j !== null); +} diff --git a/scrapper/src/scrapers/types.ts b/scrapper/src/scrapers/types.ts new file mode 100644 index 0000000..7a1d74d --- /dev/null +++ b/scrapper/src/scrapers/types.ts @@ -0,0 +1,24 @@ +/** A single job posting as returned by a scraper. */ +export interface ScrapedJob { + /** Stable ID unique within the source (used to detect new vs. already-known jobs). */ + externalId: string; + title: string; + url: string; + company?: string; + location?: string; + tags?: string[]; + /** ISO 8601 date the job was posted, if the site exposes it. */ + postedAt?: string; +} + +/** + * Implement this interface to add a new site. + * Then register the instance in `src/scrapers/index.ts`. + */ +export interface Scraper { + /** Unique, stable identifier stored in the DB (e.g. "slo-tech-ljubljana"). Don't change it later. */ + id: string; + /** Human-readable name shown in the UI. */ + name: string; + scrape(): Promise; +} diff --git a/scrapper/src/server.ts b/scrapper/src/server.ts new file mode 100644 index 0000000..b793b0c --- /dev/null +++ b/scrapper/src/server.ts @@ -0,0 +1,230 @@ +import { counts, lastRunAt, listJobs, markAllSeen, recentRuns, setSeen, sources, type Filter, type JobRow } from "./db"; +import { isScraping, runAllScrapers } from "./scrape"; +import { scrapers } from "./scrapers"; + +const PORT = Number(process.env.PORT ?? 3000); +const SCRAPE_INTERVAL_HOURS = Number(process.env.SCRAPE_INTERVAL_HOURS ?? 24); + +// --- Scheduler: scrape every SCRAPE_INTERVAL_HOURS. On startup, wait out whatever is left of the +// interval since the last run (or scrape right away if it's overdue). --- +const INTERVAL_MS = SCRAPE_INTERVAL_HOURS * 3600_000; +async function scheduledScrape() { + await runAllScrapers(); + setTimeout(scheduledScrape, INTERVAL_MS); +} +const last = lastRunAt(); +setTimeout(scheduledScrape, last ? Math.max(0, new Date(last).getTime() + INTERVAL_MS - Date.now()) : 0); + +// --- HTML rendering --- +const esc = (s: unknown) => + String(s ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c]!); + +const fmtDate = (iso: string | null) => + iso ? new Date(iso).toLocaleDateString("sl-SI", { day: "numeric", month: "short", year: "numeric" }) : ""; +const fmtDateTime = (iso: string | null) => + iso ? new Date(iso).toLocaleString("sl-SI", { dateStyle: "medium", timeStyle: "short" }) : "never"; + +const sourceName = (id: string) => scrapers.find((s) => s.id === id)?.name ?? id; + +function jobCard(j: JobRow) { + const tags = JSON.parse(j.tags) as string[]; + return ` +
  • +
    +
    + ${j.seen_at ? "" : `NEW`} + ${j.still_listed ? "" : `no longer listed`} + ${esc(j.title)} +
    +
    + ${j.company ? `${esc(j.company)}` : ""} + ${j.location ? `${esc(j.location)}` : ""} + ${esc(sourceName(j.source))} + ${j.posted_at ? `posted ${fmtDate(j.posted_at)}` : ""} + found ${fmtDate(j.first_seen_at)} +
    + ${tags.length ? `
    ${tags.map((t) => `${esc(t)}`).join("")}
    ` : ""} +
    + +
    • `; +} + +function page(url: URL) { + const filter = (["new", "seen", "all"].includes(url.searchParams.get("filter") ?? "") + ? url.searchParams.get("filter") + : "new") as Filter; + const source = url.searchParams.get("source") ?? ""; + const q = url.searchParams.get("q") ?? ""; + const jobs = listJobs({ filter, source: source || undefined, q: q || undefined }); + const c = counts(); + const runs = recentRuns(5); + const link = (f: Filter) => { + const p = new URLSearchParams(url.searchParams); + p.set("filter", f); + return `?${p}`; + }; + + return ` + + + + +Job Board + + + +
    +
    +
    +

    Job Board

    +
    Last scrape: ${fmtDateTime(lastRunAt())}${isScraping() ? " · scraping now…" : ""}
    +
    + +
    + +
    + +
    + + + +
    + ${filter !== "seen" && jobs.some((j) => !j.seen_at) ? `` : ""} +
    + + ${jobs.length ? `
      ${jobs.map(jobCard).join("")}
    ` : `

    ${filter === "new" ? "No new jobs. 🎉" : "No jobs."}

    `} + +
    + Recent scrape runs + + + ${runs.map((r) => ``).join("")} +
    SourceStartedFoundNewError
    ${esc(sourceName(r.source))}${fmtDateTime(r.started_at)}${r.found ?? ""}${r.new_count ?? ""}${esc(r.error)}
    +
    +
    + + +`; +} + +const html = (body: string) => new Response(body, { headers: { "content-type": "text/html; charset=utf-8" } }); + +Bun.serve({ + port: PORT, + routes: { + "/": (req) => html(page(new URL(req.url))), + "/api/scrape": { POST: async () => Response.json(await runAllScrapers()) }, + "/api/jobs/seen": { + POST: async (req) => { + const { ids } = (await req.json()) as { ids: number[] }; + markAllSeen(ids.filter(Number.isInteger)); + return Response.json(counts()); + }, + }, + "/api/jobs/:id/seen": { + POST: async (req) => { + const { seen } = (await req.json()) as { seen: boolean }; + setSeen(Number(req.params.id), !!seen); + return Response.json(counts()); + }, + }, + }, + fetch: () => new Response("Not found", { status: 404 }), +}); + +console.log(`Job board running at http://localhost:${PORT} (scraping every ${SCRAPE_INTERVAL_HOURS}h)`); diff --git a/scrapper/tsconfig.json b/scrapper/tsconfig.json new file mode 100644 index 0000000..a2929a0 --- /dev/null +++ b/scrapper/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "Preserve", + "moduleResolution": "bundler", + "types": ["bun"], + "strict": true, + "noEmit": true, + "skipLibCheck": true + } +}