Add job board scraper: modular scrapers, SQLite storage, web UI
Scrapes slo-tech.com (Ljubljana) every 24h, stores jobs in SQLite and tracks which are new vs. seen. New sites can be added as modules in src/scrapers/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,2 @@
|
||||
node_modules
|
||||
data/
|
||||
@@ -0,0 +1,41 @@
|
||||
# Job Board
|
||||
|
||||
Scrapes job sites once a day, stores results in SQLite (`data/jobs.sqlite`), and shows them in a web UI where you can mark jobs as seen.
|
||||
|
||||
No runtime dependencies. It uses Bun's built-in SQLite, HTTP server and HTMLRewriter.
|
||||
|
||||
## Run
|
||||
|
||||
```sh
|
||||
bun install
|
||||
bun start # http://localhost:3000
|
||||
```
|
||||
|
||||
The server scrapes every 24h. On startup it waits out the rest of the 24h since the last scrape (or scrapes right away if none happened in the last 24h).
|
||||
You can also click **Scrape now** in the UI, or run `bun run scrape` once (e.g. from cron).
|
||||
|
||||
Env vars: `PORT` (3000), `SCRAPE_INTERVAL_HOURS` (24), `DB_PATH` (`data/jobs.sqlite`).
|
||||
|
||||
## UI
|
||||
|
||||
- **New / Seen / All** tabs. Opening a job, or clicking *Mark seen*, moves it to Seen.
|
||||
- **no longer listed** means the job was missing from that source's latest successful scrape.
|
||||
- Search filters on title, company and tags. Recent scrape runs (and errors) are listed at the bottom.
|
||||
|
||||
## Adding a site
|
||||
|
||||
1. Create `src/scrapers/<site>.ts` exporting an object that implements `Scraper` (`src/scrapers/types.ts`):
|
||||
```ts
|
||||
export const mySite: Scraper = {
|
||||
id: "my-site", // stable, stored in DB
|
||||
name: "My Site",
|
||||
async scrape() {
|
||||
const html = await fetchHtml("https://..."); // helper in ./http.ts
|
||||
return [/* { externalId, title, url, company?, location?, tags?, postedAt? } */];
|
||||
},
|
||||
};
|
||||
```
|
||||
2. Add it to the list in `src/scrapers/index.ts`.
|
||||
|
||||
`externalId` must stay stable for a job, because it is what decides whether a job is new.
|
||||
For another slo-tech city, just add `sloTech("Maribor")` to the list.
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "job-board",
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@types/bun": ["@types/bun@1.4.3", "", { "dependencies": { "bun-types": "1.4.3" } }, "sha512-N67Pqy8JXbAY/j0tNraS3V5mubGvc6rrrNDmDCtZLlFzccbUJPDb0tvgUBC5q4m8ybkP93Jcn1QB+RYQ6PvBhw=="],
|
||||
|
||||
"@types/node": ["@types/node@26.6.5", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg=="],
|
||||
|
||||
"bun-types": ["bun-types@1.4.3", "", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-tinRJNGOmUN+tEAsn2y2e942n9JHRHR42iUYzDH75NmhJqVBokGSQkkfSFdNWwsANIdrK6ACH9t01M452YyTwg=="],
|
||||
|
||||
"undici-types": ["undici-types@8.11.2", "", {}, "sha512-iMVNmWZ0leK/goS6eXMizSzmm9CDWtyphwbaCms3DNLqRxDL+mMoNVcZMTyyVgXP0N+Z8neAMzDoUOUJL8veKg=="],
|
||||
|
||||
"@types/node/undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "job-board",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "bun run src/server.ts",
|
||||
"dev": "bun --watch run src/server.ts",
|
||||
"scrape": "bun run src/scrape.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import type { ScrapedJob } from "./scrapers/types";
|
||||
|
||||
const DB_PATH = process.env.DB_PATH ?? `${import.meta.dir}/../data/jobs.sqlite`;
|
||||
mkdirSync(dirname(DB_PATH), { recursive: true });
|
||||
|
||||
export const db = new Database(DB_PATH, { create: true, strict: true });
|
||||
db.run("PRAGMA journal_mode = WAL");
|
||||
db.run(`
|
||||
CREATE TABLE IF NOT EXISTS jobs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
external_id TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
company TEXT,
|
||||
url TEXT NOT NULL,
|
||||
location TEXT,
|
||||
tags TEXT NOT NULL DEFAULT '[]',
|
||||
posted_at TEXT,
|
||||
first_seen_at TEXT NOT NULL, -- when the scraper first found it
|
||||
last_seen_at TEXT NOT NULL, -- last scrape that still listed it
|
||||
seen_at TEXT, -- when you marked it as seen (NULL = new)
|
||||
UNIQUE (source, external_id)
|
||||
)
|
||||
`);
|
||||
db.run(`
|
||||
CREATE TABLE IF NOT EXISTS scrape_runs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT,
|
||||
found INTEGER,
|
||||
new_count INTEGER,
|
||||
error TEXT
|
||||
)
|
||||
`);
|
||||
|
||||
export interface JobRow {
|
||||
id: number;
|
||||
source: string;
|
||||
external_id: string;
|
||||
title: string;
|
||||
company: string | null;
|
||||
url: string;
|
||||
location: string | null;
|
||||
tags: string;
|
||||
posted_at: string | null;
|
||||
first_seen_at: string;
|
||||
last_seen_at: string;
|
||||
seen_at: string | null;
|
||||
/** 0 if the job was missing from the source's latest successful scrape. */
|
||||
still_listed: number;
|
||||
}
|
||||
|
||||
export interface RunRow {
|
||||
id: number;
|
||||
source: string;
|
||||
started_at: string;
|
||||
finished_at: string | null;
|
||||
found: number | null;
|
||||
new_count: number | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
const upsertJob = db.prepare(`
|
||||
INSERT INTO jobs (source, external_id, title, company, url, location, tags, posted_at, first_seen_at, last_seen_at)
|
||||
VALUES ($source, $externalId, $title, $company, $url, $location, $tags, $postedAt, $now, $now)
|
||||
ON CONFLICT (source, external_id) DO UPDATE SET
|
||||
title = excluded.title, company = excluded.company, url = excluded.url, location = excluded.location,
|
||||
tags = excluded.tags, posted_at = excluded.posted_at, last_seen_at = excluded.last_seen_at
|
||||
RETURNING first_seen_at = $now AS is_new
|
||||
`);
|
||||
|
||||
/** Insert or refresh scraped jobs; returns how many were not in the DB before. */
|
||||
export const saveJobs = db.transaction((source: string, jobs: ScrapedJob[], now: string): number => {
|
||||
let newCount = 0;
|
||||
for (const j of jobs) {
|
||||
const row = upsertJob.get({
|
||||
source,
|
||||
externalId: j.externalId,
|
||||
title: j.title,
|
||||
company: j.company ?? null,
|
||||
url: j.url,
|
||||
location: j.location ?? null,
|
||||
tags: JSON.stringify(j.tags ?? []),
|
||||
postedAt: j.postedAt ?? null,
|
||||
now,
|
||||
}) as { is_new: number };
|
||||
newCount += row.is_new;
|
||||
}
|
||||
return newCount;
|
||||
});
|
||||
|
||||
export function startRun(source: string, now: string): number {
|
||||
const row = db
|
||||
.query("INSERT INTO scrape_runs (source, started_at) VALUES (?, ?) RETURNING id")
|
||||
.get(source, now) as { id: number };
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export function finishRun(id: number, result: { found?: number; newCount?: number; error?: string }) {
|
||||
db.query("UPDATE scrape_runs SET finished_at = ?, found = ?, new_count = ?, error = ? WHERE id = ?").run(
|
||||
new Date().toISOString(),
|
||||
result.found ?? null,
|
||||
result.newCount ?? null,
|
||||
result.error ?? null,
|
||||
id,
|
||||
);
|
||||
}
|
||||
|
||||
export type Filter = "new" | "seen" | "all";
|
||||
|
||||
export function listJobs(opts: { filter: Filter; source?: string; q?: string }): JobRow[] {
|
||||
const where: string[] = [];
|
||||
const params: Record<string, string> = {};
|
||||
if (opts.filter === "new") where.push("j.seen_at IS NULL");
|
||||
if (opts.filter === "seen") where.push("j.seen_at IS NOT NULL");
|
||||
if (opts.source) {
|
||||
where.push("j.source = $source");
|
||||
params.source = opts.source;
|
||||
}
|
||||
if (opts.q) {
|
||||
where.push("(j.title LIKE $q OR j.company LIKE $q OR j.tags LIKE $q)");
|
||||
params.q = `%${opts.q}%`;
|
||||
}
|
||||
return db
|
||||
.query(
|
||||
`SELECT j.*,
|
||||
j.last_seen_at >= COALESCE(
|
||||
(SELECT MAX(r.started_at) FROM scrape_runs r WHERE r.source = j.source AND r.error IS NULL AND r.finished_at IS NOT NULL),
|
||||
j.last_seen_at
|
||||
) AS still_listed
|
||||
FROM jobs j
|
||||
${where.length ? "WHERE " + where.join(" AND ") : ""}
|
||||
ORDER BY still_listed DESC, COALESCE(j.posted_at, j.first_seen_at) DESC`,
|
||||
)
|
||||
.all(params) as JobRow[];
|
||||
}
|
||||
|
||||
export function counts(): { new: number; seen: number; all: number } {
|
||||
return db
|
||||
.query(
|
||||
`SELECT SUM(seen_at IS NULL) AS new, SUM(seen_at IS NOT NULL) AS seen, COUNT(*) AS "all" FROM jobs`,
|
||||
)
|
||||
.get() as { new: number; seen: number; all: number };
|
||||
}
|
||||
|
||||
export function setSeen(id: number, seen: boolean) {
|
||||
db.query("UPDATE jobs SET seen_at = ? WHERE id = ?").run(seen ? new Date().toISOString() : null, id);
|
||||
}
|
||||
|
||||
export function markAllSeen(ids: number[]) {
|
||||
const stmt = db.query("UPDATE jobs SET seen_at = ? WHERE id = ? AND seen_at IS NULL");
|
||||
const now = new Date().toISOString();
|
||||
db.transaction(() => ids.forEach((id) => stmt.run(now, id)))();
|
||||
}
|
||||
|
||||
export function sources(): string[] {
|
||||
return (db.query("SELECT DISTINCT source FROM jobs ORDER BY source").all() as { source: string }[]).map(
|
||||
(r) => r.source,
|
||||
);
|
||||
}
|
||||
|
||||
export function recentRuns(limit = 10): RunRow[] {
|
||||
return db.query("SELECT * FROM scrape_runs ORDER BY id DESC LIMIT ?").all(limit) as RunRow[];
|
||||
}
|
||||
|
||||
/** Start time of the most recent run that finished (successfully or not), across all sources. */
|
||||
export function lastRunAt(): string | null {
|
||||
const row = db.query("SELECT MAX(started_at) AS t FROM scrape_runs WHERE finished_at IS NOT NULL").get() as {
|
||||
t: string | null;
|
||||
};
|
||||
return row.t;
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { finishRun, saveJobs, startRun } from "./db";
|
||||
import { scrapers } from "./scrapers";
|
||||
|
||||
export interface RunSummary {
|
||||
source: string;
|
||||
found?: number;
|
||||
newCount?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
let running: Promise<RunSummary[]> | null = null;
|
||||
|
||||
/** Run every registered scraper. A failing scraper is logged and doesn't stop the others. */
|
||||
export function runAllScrapers(): Promise<RunSummary[]> {
|
||||
running ??= (async () => {
|
||||
const results: RunSummary[] = [];
|
||||
for (const s of scrapers) {
|
||||
const now = new Date().toISOString();
|
||||
const runId = startRun(s.id, now);
|
||||
try {
|
||||
const jobs = await s.scrape();
|
||||
if (jobs.length === 0) throw new Error("scraper returned 0 jobs (site layout changed?)");
|
||||
const newCount = saveJobs(s.id, jobs, now);
|
||||
finishRun(runId, { found: jobs.length, newCount });
|
||||
results.push({ source: s.id, found: jobs.length, newCount });
|
||||
console.log(`[scrape] ${s.id}: ${jobs.length} found, ${newCount} new`);
|
||||
} catch (e) {
|
||||
const error = e instanceof Error ? e.message : String(e);
|
||||
finishRun(runId, { error });
|
||||
results.push({ source: s.id, error });
|
||||
console.error(`[scrape] ${s.id} failed: ${error}`);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
})().finally(() => (running = null));
|
||||
return running;
|
||||
}
|
||||
|
||||
export const isScraping = () => running !== null;
|
||||
|
||||
if (import.meta.main) {
|
||||
const results = await runAllScrapers();
|
||||
process.exit(results.some((r) => r.error) ? 1 : 0);
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
// Shared helpers for scrapers.
|
||||
|
||||
const USER_AGENT =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36";
|
||||
|
||||
// Upper half (0xA0-0xFF) of ISO-8859-2, which Bun's TextDecoder does not support.
|
||||
const LATIN2_HIGH =
|
||||
" Ą˘Ł¤ĽŚ§¨ŠŞŤŹŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
|
||||
|
||||
function decodeLatin2(bytes: Uint8Array): string {
|
||||
let out = "";
|
||||
for (const b of bytes) out += b < 0xa0 ? String.fromCharCode(b) : LATIN2_HIGH[b - 0xa0];
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Fetch a page and decode it according to its declared charset. */
|
||||
export async function fetchHtml(url: string): Promise<string> {
|
||||
const res = await fetch(url, { headers: { "User-Agent": USER_AGENT } });
|
||||
if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`);
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
const charset = /charset=([\w-]+)/i.exec(res.headers.get("content-type") ?? "")?.[1]?.toLowerCase() ?? "utf-8";
|
||||
if (charset === "iso-8859-2" || charset === "latin2") return decodeLatin2(bytes);
|
||||
try {
|
||||
return new TextDecoder(charset).decode(bytes);
|
||||
} catch {
|
||||
return new TextDecoder("utf-8").decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
const NAMED_ENTITIES: Record<string, string> = {
|
||||
amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", raquo: "»", laquo: "«",
|
||||
};
|
||||
|
||||
/** Decode HTML entities (HTMLRewriter text chunks are not decoded). */
|
||||
export function decodeEntities(s: string): string {
|
||||
return s.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (m, e: string) => {
|
||||
if (e[0] === "#") {
|
||||
const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
|
||||
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return NAMED_ENTITIES[e.toLowerCase()] ?? m;
|
||||
});
|
||||
}
|
||||
|
||||
export const clean = (s: string) => decodeEntities(s).replace(/\s+/g, " ").trim();
|
||||
@@ -0,0 +1,13 @@
|
||||
import { sloTech } from "./slotech";
|
||||
import type { Scraper } from "./types";
|
||||
|
||||
/**
|
||||
* All active scrapers. To add a site:
|
||||
* 1. create src/scrapers/<site>.ts exporting an object implementing `Scraper`
|
||||
* 2. add it to this list
|
||||
*/
|
||||
export const scrapers: Scraper[] = [
|
||||
sloTech("Ljubljana"),
|
||||
];
|
||||
|
||||
export type { Scraper, ScrapedJob } from "./types";
|
||||
@@ -0,0 +1,74 @@
|
||||
import { clean, fetchHtml } from "./http";
|
||||
import type { ScrapedJob, Scraper } from "./types";
|
||||
|
||||
const BASE = "https://slo-tech.com";
|
||||
|
||||
/** Scraper for slo-tech.com job listings, filtered by city (e.g. "Ljubljana", "Maribor"). */
|
||||
export function sloTech(city: string): Scraper {
|
||||
return {
|
||||
id: `slo-tech-${city.toLowerCase()}`,
|
||||
name: `Slo-Tech (${city})`,
|
||||
async scrape() {
|
||||
const html = await fetchHtml(`${BASE}/delo/mesto/${encodeURIComponent(city)}`);
|
||||
return parseSloTech(html, city);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function parseSloTech(html: string, city: string): Promise<ScrapedJob[]> {
|
||||
type Draft = { href: string; title: string; company: string; tags: string[]; postedAt?: string };
|
||||
const drafts: Draft[] = [];
|
||||
let cur: Draft | undefined;
|
||||
|
||||
await new HTMLRewriter()
|
||||
.on("table.forums tbody tr", {
|
||||
element() {
|
||||
cur = { href: "", title: "", company: "", tags: [] };
|
||||
drafts.push(cur);
|
||||
},
|
||||
})
|
||||
.on("table.forums tbody td.name h3 a", {
|
||||
element(el) {
|
||||
if (cur) cur.href = el.getAttribute("href") ?? "";
|
||||
},
|
||||
text(t) {
|
||||
if (cur) cur.title += t.text;
|
||||
},
|
||||
})
|
||||
.on("table.forums tbody td.name span.oddelek a", {
|
||||
element() {
|
||||
cur?.tags.push("");
|
||||
},
|
||||
text(t) {
|
||||
if (cur?.tags.length) cur.tags[cur.tags.length - 1] += t.text;
|
||||
},
|
||||
})
|
||||
.on("table.forums tbody td.company", {
|
||||
text(t) {
|
||||
if (cur) cur.company += t.text;
|
||||
},
|
||||
})
|
||||
.on("table.forums tbody td.last_msg time", {
|
||||
element(el) {
|
||||
if (cur) cur.postedAt = el.getAttribute("datetime") ?? undefined;
|
||||
},
|
||||
})
|
||||
.transform(new Response(html))
|
||||
.text();
|
||||
|
||||
return drafts
|
||||
.map((d): ScrapedJob | null => {
|
||||
const id = /\/delo\/(\d+)/.exec(d.href)?.[1];
|
||||
if (!id) return null;
|
||||
return {
|
||||
externalId: id,
|
||||
title: clean(d.title),
|
||||
url: new URL(d.href, BASE).href,
|
||||
company: clean(d.company) || undefined,
|
||||
location: city,
|
||||
tags: d.tags.map(clean).filter(Boolean),
|
||||
postedAt: d.postedAt ? new Date(d.postedAt).toISOString() : undefined,
|
||||
};
|
||||
})
|
||||
.filter((j): j is ScrapedJob => j !== null);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
/** A single job posting as returned by a scraper. */
|
||||
export interface ScrapedJob {
|
||||
/** Stable ID unique within the source (used to detect new vs. already-known jobs). */
|
||||
externalId: string;
|
||||
title: string;
|
||||
url: string;
|
||||
company?: string;
|
||||
location?: string;
|
||||
tags?: string[];
|
||||
/** ISO 8601 date the job was posted, if the site exposes it. */
|
||||
postedAt?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Implement this interface to add a new site.
|
||||
* Then register the instance in `src/scrapers/index.ts`.
|
||||
*/
|
||||
export interface Scraper {
|
||||
/** Unique, stable identifier stored in the DB (e.g. "slo-tech-ljubljana"). Don't change it later. */
|
||||
id: string;
|
||||
/** Human-readable name shown in the UI. */
|
||||
name: string;
|
||||
scrape(): Promise<ScrapedJob[]>;
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
import { counts, lastRunAt, listJobs, markAllSeen, recentRuns, setSeen, sources, type Filter, type JobRow } from "./db";
|
||||
import { isScraping, runAllScrapers } from "./scrape";
|
||||
import { scrapers } from "./scrapers";
|
||||
|
||||
const PORT = Number(process.env.PORT ?? 3000);
|
||||
const SCRAPE_INTERVAL_HOURS = Number(process.env.SCRAPE_INTERVAL_HOURS ?? 24);
|
||||
|
||||
// --- Scheduler: scrape every SCRAPE_INTERVAL_HOURS. On startup, wait out whatever is left of the
|
||||
// interval since the last run (or scrape right away if it's overdue). ---
|
||||
const INTERVAL_MS = SCRAPE_INTERVAL_HOURS * 3600_000;
|
||||
async function scheduledScrape() {
|
||||
await runAllScrapers();
|
||||
setTimeout(scheduledScrape, INTERVAL_MS);
|
||||
}
|
||||
const last = lastRunAt();
|
||||
setTimeout(scheduledScrape, last ? Math.max(0, new Date(last).getTime() + INTERVAL_MS - Date.now()) : 0);
|
||||
|
||||
// --- HTML rendering ---
|
||||
const esc = (s: unknown) =>
|
||||
String(s ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c]!);
|
||||
|
||||
const fmtDate = (iso: string | null) =>
|
||||
iso ? new Date(iso).toLocaleDateString("sl-SI", { day: "numeric", month: "short", year: "numeric" }) : "";
|
||||
const fmtDateTime = (iso: string | null) =>
|
||||
iso ? new Date(iso).toLocaleString("sl-SI", { dateStyle: "medium", timeStyle: "short" }) : "never";
|
||||
|
||||
const sourceName = (id: string) => scrapers.find((s) => s.id === id)?.name ?? id;
|
||||
|
||||
function jobCard(j: JobRow) {
|
||||
const tags = JSON.parse(j.tags) as string[];
|
||||
return `
|
||||
<li class="job ${j.seen_at ? "seen" : "new"} ${j.still_listed ? "" : "gone"}" data-id="${j.id}">
|
||||
<div class="main">
|
||||
<div class="title-row">
|
||||
${j.seen_at ? "" : `<span class="badge new-badge">NEW</span>`}
|
||||
${j.still_listed ? "" : `<span class="badge gone-badge" title="Missing from the latest scrape">no longer listed</span>`}
|
||||
<a class="title" href="${esc(j.url)}" target="_blank" rel="noopener">${esc(j.title)}</a>
|
||||
</div>
|
||||
<div class="meta">
|
||||
${j.company ? `<span class="company">${esc(j.company)}</span>` : ""}
|
||||
${j.location ? `<span>${esc(j.location)}</span>` : ""}
|
||||
<span>${esc(sourceName(j.source))}</span>
|
||||
${j.posted_at ? `<span title="Posted">posted ${fmtDate(j.posted_at)}</span>` : ""}
|
||||
<span title="First found by scraper">found ${fmtDate(j.first_seen_at)}</span>
|
||||
</div>
|
||||
${tags.length ? `<div class="tags">${tags.map((t) => `<span class="tag">${esc(t)}</span>`).join("")}</div>` : ""}
|
||||
</div>
|
||||
<button class="toggle" type="button">${j.seen_at ? "Mark new" : "Mark seen"}</button>
|
||||
</li>`;
|
||||
}
|
||||
|
||||
function page(url: URL) {
|
||||
const filter = (["new", "seen", "all"].includes(url.searchParams.get("filter") ?? "")
|
||||
? url.searchParams.get("filter")
|
||||
: "new") as Filter;
|
||||
const source = url.searchParams.get("source") ?? "";
|
||||
const q = url.searchParams.get("q") ?? "";
|
||||
const jobs = listJobs({ filter, source: source || undefined, q: q || undefined });
|
||||
const c = counts();
|
||||
const runs = recentRuns(5);
|
||||
const link = (f: Filter) => {
|
||||
const p = new URLSearchParams(url.searchParams);
|
||||
p.set("filter", f);
|
||||
return `?${p}`;
|
||||
};
|
||||
|
||||
return `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Job Board</title>
|
||||
<style>
|
||||
:root { --bg:#f6f7f9; --card:#fff; --fg:#1c1f24; --muted:#6b7280; --border:#e3e6ea; --accent:#2563eb; --new:#16a34a; --gone:#b45309; --tag:#eef1f5; }
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root { --bg:#111317; --card:#1a1d23; --fg:#e6e8eb; --muted:#9aa1ab; --border:#2a2e36; --accent:#60a5fa; --new:#4ade80; --gone:#fbbf24; --tag:#252a32; }
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin:0; background:var(--bg); color:var(--fg); font:15px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; }
|
||||
.wrap { max-width: 920px; margin: 0 auto; padding: 24px 16px 64px; }
|
||||
header { display:flex; flex-wrap:wrap; align-items:center; gap:12px; justify-content:space-between; margin-bottom:16px; }
|
||||
h1 { font-size:22px; margin:0; }
|
||||
.status { color:var(--muted); font-size:13px; }
|
||||
button, .btn { font:inherit; font-size:13px; padding:6px 12px; border-radius:8px; border:1px solid var(--border); background:var(--card); color:var(--fg); cursor:pointer; }
|
||||
button:hover { border-color:var(--accent); }
|
||||
button:disabled { opacity:.6; cursor:wait; }
|
||||
.toolbar { display:flex; flex-wrap:wrap; gap:8px; align-items:center; margin-bottom:16px; }
|
||||
.tabs { display:flex; gap:4px; background:var(--card); border:1px solid var(--border); border-radius:10px; padding:3px; }
|
||||
.tabs a { padding:5px 12px; border-radius:7px; color:var(--fg); text-decoration:none; font-size:14px; }
|
||||
.tabs a.active { background:var(--accent); color:#fff; }
|
||||
.toolbar form { display:flex; gap:8px; flex:1; min-width:240px; }
|
||||
input, select { font:inherit; font-size:14px; padding:6px 10px; border-radius:8px; border:1px solid var(--border); background:var(--card); color:var(--fg); }
|
||||
input[type=search] { flex:1; min-width:0; }
|
||||
ul.jobs { list-style:none; padding:0; margin:0; display:flex; flex-direction:column; gap:8px; }
|
||||
.job { display:flex; gap:12px; align-items:flex-start; background:var(--card); border:1px solid var(--border); border-left:4px solid var(--new); border-radius:10px; padding:12px 14px; }
|
||||
.job.seen { border-left-color:var(--border); }
|
||||
.job.seen .title { color:var(--muted); }
|
||||
.job.gone { opacity:.65; }
|
||||
.main { flex:1; min-width:0; }
|
||||
.title-row { display:flex; flex-wrap:wrap; align-items:center; gap:6px; }
|
||||
.title { font-weight:600; font-size:16px; color:var(--fg); text-decoration:none; overflow-wrap:anywhere; }
|
||||
.title:hover { color:var(--accent); text-decoration:underline; }
|
||||
.badge { font-size:11px; font-weight:700; padding:1px 6px; border-radius:5px; }
|
||||
.new-badge { color:var(--new); border:1px solid var(--new); }
|
||||
.gone-badge { color:var(--gone); border:1px solid var(--gone); font-weight:500; }
|
||||
.meta { display:flex; flex-wrap:wrap; gap:4px 12px; color:var(--muted); font-size:13px; margin-top:2px; }
|
||||
.company { color:var(--fg); font-weight:500; }
|
||||
.tags { display:flex; flex-wrap:wrap; gap:4px; margin-top:6px; }
|
||||
.tag { background:var(--tag); font-size:12px; padding:1px 8px; border-radius:999px; max-width:100%; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
|
||||
.toggle { flex-shrink:0; }
|
||||
.empty { text-align:center; color:var(--muted); padding:48px 0; }
|
||||
details { margin-top:32px; color:var(--muted); font-size:13px; }
|
||||
table { border-collapse:collapse; width:100%; margin-top:8px; }
|
||||
td, th { text-align:left; padding:4px 8px; border-bottom:1px solid var(--border); }
|
||||
.err { color:#dc2626; }
|
||||
@media (max-width: 560px) { .job { flex-direction:column; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<header>
|
||||
<div>
|
||||
<h1>Job Board</h1>
|
||||
<div class="status">Last scrape: ${fmtDateTime(lastRunAt())}${isScraping() ? " · scraping now…" : ""}</div>
|
||||
</div>
|
||||
<button id="scrape" type="button" ${isScraping() ? "disabled" : ""}>Scrape now</button>
|
||||
</header>
|
||||
|
||||
<div class="toolbar">
|
||||
<nav class="tabs">
|
||||
<a href="${link("new")}" class="${filter === "new" ? "active" : ""}">New (<span id="c-new">${c.new ?? 0}</span>)</a>
|
||||
<a href="${link("seen")}" class="${filter === "seen" ? "active" : ""}">Seen (<span id="c-seen">${c.seen ?? 0}</span>)</a>
|
||||
<a href="${link("all")}" class="${filter === "all" ? "active" : ""}">All (${c.all ?? 0})</a>
|
||||
</nav>
|
||||
<form method="get">
|
||||
<input type="hidden" name="filter" value="${filter}">
|
||||
<input type="search" name="q" value="${esc(q)}" placeholder="Search title, company, tags…">
|
||||
<select name="source" onchange="this.form.submit()">
|
||||
<option value="">All sources</option>
|
||||
${sources().map((s) => `<option value="${esc(s)}" ${s === source ? "selected" : ""}>${esc(sourceName(s))}</option>`).join("")}
|
||||
</select>
|
||||
</form>
|
||||
${filter !== "seen" && jobs.some((j) => !j.seen_at) ? `<button id="all-seen" type="button">Mark all shown as seen</button>` : ""}
|
||||
</div>
|
||||
|
||||
${jobs.length ? `<ul class="jobs">${jobs.map(jobCard).join("")}</ul>` : `<p class="empty">${filter === "new" ? "No new jobs. 🎉" : "No jobs."}</p>`}
|
||||
|
||||
<details>
|
||||
<summary>Recent scrape runs</summary>
|
||||
<table>
|
||||
<tr><th>Source</th><th>Started</th><th>Found</th><th>New</th><th>Error</th></tr>
|
||||
${runs.map((r) => `<tr><td>${esc(sourceName(r.source))}</td><td>${fmtDateTime(r.started_at)}</td><td>${r.found ?? ""}</td><td>${r.new_count ?? ""}</td><td class="err">${esc(r.error)}</td></tr>`).join("")}
|
||||
</table>
|
||||
</details>
|
||||
</div>
|
||||
<script>
|
||||
const filter = ${JSON.stringify(filter)};
|
||||
async function setSeen(li, seen) {
|
||||
const res = await fetch("/api/jobs/" + li.dataset.id + "/seen", {
|
||||
method: "POST", headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ seen }), keepalive: true,
|
||||
});
|
||||
const c = await res.json();
|
||||
document.getElementById("c-new").textContent = c.new ?? 0;
|
||||
document.getElementById("c-seen").textContent = c.seen ?? 0;
|
||||
li.classList.toggle("seen", seen);
|
||||
li.classList.toggle("new", !seen);
|
||||
li.querySelector(".new-badge")?.remove();
|
||||
if (!seen) li.querySelector(".title-row").insertAdjacentHTML("afterbegin", '<span class="badge new-badge">NEW</span>');
|
||||
li.querySelector(".toggle").textContent = seen ? "Mark new" : "Mark seen";
|
||||
if ((filter === "new" && seen) || (filter === "seen" && !seen)) li.remove();
|
||||
}
|
||||
document.querySelectorAll(".job").forEach((li) => {
|
||||
li.querySelector(".toggle").addEventListener("click", () => setSeen(li, !li.classList.contains("seen")));
|
||||
// Opening a job counts as seeing it (the card stays until you reload).
|
||||
li.querySelector(".title").addEventListener("click", () => {
|
||||
if (li.classList.contains("seen")) return;
|
||||
fetch("/api/jobs/" + li.dataset.id + "/seen", {
|
||||
method: "POST", headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ seen: true }), keepalive: true,
|
||||
});
|
||||
li.classList.add("seen"); li.classList.remove("new");
|
||||
li.querySelector(".new-badge")?.remove();
|
||||
li.querySelector(".toggle").textContent = "Mark new";
|
||||
});
|
||||
});
|
||||
document.getElementById("all-seen")?.addEventListener("click", async () => {
|
||||
const ids = [...document.querySelectorAll(".job.new")].map((li) => Number(li.dataset.id));
|
||||
await fetch("/api/jobs/seen", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ ids }) });
|
||||
location.reload();
|
||||
});
|
||||
document.getElementById("scrape").addEventListener("click", async (e) => {
|
||||
e.target.disabled = true; e.target.textContent = "Scraping…";
|
||||
const res = await fetch("/api/scrape", { method: "POST" });
|
||||
const results = await res.json();
|
||||
const errors = results.filter((r) => r.error);
|
||||
if (errors.length) alert("Some scrapers failed:\\n" + errors.map((r) => r.source + ": " + r.error).join("\\n"));
|
||||
location.reload();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
const html = (body: string) => new Response(body, { headers: { "content-type": "text/html; charset=utf-8" } });
|
||||
|
||||
Bun.serve({
|
||||
port: PORT,
|
||||
routes: {
|
||||
"/": (req) => html(page(new URL(req.url))),
|
||||
"/api/scrape": { POST: async () => Response.json(await runAllScrapers()) },
|
||||
"/api/jobs/seen": {
|
||||
POST: async (req) => {
|
||||
const { ids } = (await req.json()) as { ids: number[] };
|
||||
markAllSeen(ids.filter(Number.isInteger));
|
||||
return Response.json(counts());
|
||||
},
|
||||
},
|
||||
"/api/jobs/:id/seen": {
|
||||
POST: async (req) => {
|
||||
const { seen } = (await req.json()) as { seen: boolean };
|
||||
setSeen(Number(req.params.id), !!seen);
|
||||
return Response.json(counts());
|
||||
},
|
||||
},
|
||||
},
|
||||
fetch: () => new Response("Not found", { status: 404 }),
|
||||
});
|
||||
|
||||
console.log(`Job board running at http://localhost:${PORT} (scraping every ${SCRAPE_INTERVAL_HOURS}h)`);
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"lib": ["ESNext", "DOM"],
|
||||
"target": "ESNext",
|
||||
"module": "Preserve",
|
||||
"moduleResolution": "bundler",
|
||||
"types": ["bun"],
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"skipLibCheck": true
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user