Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
beac32469c | ||
|
|
ad2fa2b775 | ||
|
|
5d30bcf85a | ||
|
|
88420b54e7 | ||
|
|
f6e5605108 | ||
|
|
0324ff7f2a |
@@ -0,0 +1,3 @@
|
||||
node_modules
|
||||
data/
|
||||
bin/
|
||||
@@ -0,0 +1,91 @@
|
||||
# Job Board
|
||||
|
||||
Scrapes job sites once a day, stores results in SQLite (`data/jobs.sqlite`), and shows them in a web UI where you can mark jobs as seen.
|
||||
|
||||
No runtime dependencies. It uses Bun's built-in SQLite, HTTP server and HTMLRewriter.
|
||||
|
||||
## Sources
|
||||
|
||||
| Source | How | Notes |
|
||||
|---|---|---|
|
||||
| slo-tech.com (Ljubljana) | HTML scrape | ~20 newest jobs |
|
||||
| startup.si | HTML scrape (no public API or feed) | Site lists every posting since 2019 on one page, so only the last 90 days are kept (`startupSi(days)` in `src/scrapers/index.ts`) |
|
||||
| adecco.com (Slovenia, IT) | Internal JSON API the site's job search uses (no public API) | Filtered by category; add more with `adecco("<CategoryFilter value>", "<label>")` |
|
||||
| mojedelo.com (IT, Osrednjeslovenska) | Internal JSON API the site's search uses (no public API) | Takes a search URL copied from the site; add more with `mojeDelo("<label>", "<search URL>")` (don't rename the label later, it's part of the source ID) |
|
||||
| optius.com (IT, Osrednjeslovenska) | Official RSS feed | Build a feed URL at https://www.optius.com/rss/ (its IDs differ from the search page's); add more with `optius("<label>", "<feed URL>")` (same label rule as MojeDelo) |
|
||||
|
||||
## Run
|
||||
|
||||
```sh
|
||||
bun install
|
||||
bun start # http://localhost:3000
|
||||
bun start --port 8080 # or -p 8080
|
||||
```
|
||||
|
||||
The server scrapes every 24h. On startup it waits out the rest of the 24h since the last scrape (or scrapes right away if none happened in the last 24h).
|
||||
You can also click **Scrape now** in the UI, or run `bun run scrape` once (e.g. from cron).
|
||||
|
||||
Env vars: `PORT` (3000; `--port` overrides it), `SCRAPE_INTERVAL_HOURS` (24), `DB_PATH` (`data/jobs.sqlite`).
|
||||
|
||||
## Build a binary for a server
|
||||
|
||||
`bun build --compile` bundles the app and the Bun runtime into one executable (~90 MB), so the
|
||||
server doesn't need Bun or `node_modules`:
|
||||
|
||||
```sh
|
||||
bun run build # bin/job-board for this machine
|
||||
bun run build:linux-x64 # bin/job-board-linux-x64
|
||||
bun run build:linux-arm64 # bin/job-board-linux-arm64 (e.g. Raspberry Pi, ARM VPS)
|
||||
```
|
||||
|
||||
On the server:
|
||||
|
||||
```sh
|
||||
./job-board # web server on :3000, scrapes every 24h
|
||||
./job-board --port 8080
|
||||
./job-board scrape # scrape once and exit (exit code 1 if a scraper failed)
|
||||
```
|
||||
|
||||
The database goes to `data/jobs.sqlite` next to the binary (override with `DB_PATH`).
|
||||
The binaries target glibc Linux; for Alpine use `--target=bun-linux-x64-musl`, and for old CPUs
|
||||
without AVX2 use `--target=bun-linux-x64-baseline`.
|
||||
|
||||
Example systemd unit (`/etc/systemd/system/job-board.service`):
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Job board
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/job-board/job-board --port 3000
|
||||
Restart=on-failure
|
||||
User=jobboard
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
## UI
|
||||
|
||||
- **New / Seen / All** tabs. Opening a job, or clicking *Mark seen*, moves it to Seen.
|
||||
- **no longer listed** means the job was missing from that source's latest successful scrape.
|
||||
- Search filters on title, company and tags. Recent scrape runs (and errors) are listed at the bottom.
|
||||
|
||||
## Adding a site
|
||||
|
||||
1. Create `src/scrapers/<site>.ts` exporting an object that implements `Scraper` (`src/scrapers/types.ts`):
|
||||
```ts
|
||||
export const mySite: Scraper = {
|
||||
id: "my-site", // stable, stored in DB
|
||||
name: "My Site",
|
||||
async scrape() {
|
||||
const html = await fetchHtml("https://..."); // helper in ./http.ts
|
||||
return [/* { externalId, title, url, company?, location?, tags?, postedAt? } */];
|
||||
},
|
||||
};
|
||||
```
|
||||
2. Add it to the list in `src/scrapers/index.ts`.
|
||||
|
||||
`externalId` must stay stable for a job, because it is what decides whether a job is new.
|
||||
For another slo-tech city, just add `sloTech("Maribor")` to the list.
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "job-board",
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@types/bun": ["@types/bun@1.4.3", "", { "dependencies": { "bun-types": "1.4.3" } }, "sha512-N67Pqy8JXbAY/j0tNraS3V5mubGvc6rrrNDmDCtZLlFzccbUJPDb0tvgUBC5q4m8ybkP93Jcn1QB+RYQ6PvBhw=="],
|
||||
|
||||
"@types/node": ["@types/node@26.6.5", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg=="],
|
||||
|
||||
"bun-types": ["bun-types@1.4.3", "", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-tinRJNGOmUN+tEAsn2y2e942n9JHRHR42iUYzDH75NmhJqVBokGSQkkfSFdNWwsANIdrK6ACH9t01M452YyTwg=="],
|
||||
|
||||
"undici-types": ["undici-types@8.11.2", "", {}, "sha512-iMVNmWZ0leK/goS6eXMizSzmm9CDWtyphwbaCms3DNLqRxDL+mMoNVcZMTyyVgXP0N+Z8neAMzDoUOUJL8veKg=="],
|
||||
|
||||
"@types/node/undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"name": "job-board",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "bun run src/main.ts",
|
||||
"dev": "bun --watch run src/main.ts",
|
||||
"scrape": "bun run src/scrape.ts",
|
||||
"build": "bun build src/main.ts --compile --minify --sourcemap --outfile bin/job-board",
|
||||
"build:linux-x64": "bun build src/main.ts --compile --minify --sourcemap --target=bun-linux-x64 --outfile bin/job-board-linux-x64",
|
||||
"build:linux-arm64": "bun build src/main.ts --compile --minify --sourcemap --target=bun-linux-arm64 --outfile bin/job-board-linux-arm64"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
import { Database } from "bun:sqlite";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import type { ScrapedJob } from "./scrapers/types";
|
||||
|
||||
// In a compiled binary (`bun run build`) import.meta.dir is a virtual path, so keep data next to the executable.
|
||||
const isCompiled = import.meta.dir.startsWith("/$bunfs");
|
||||
const baseDir = isCompiled ? dirname(process.execPath) : `${import.meta.dir}/..`;
|
||||
const DB_PATH = process.env.DB_PATH ?? `${baseDir}/data/jobs.sqlite`;
|
||||
mkdirSync(dirname(DB_PATH), { recursive: true });
|
||||
|
||||
export const db = new Database(DB_PATH, { create: true, strict: true });
|
||||
db.run("PRAGMA journal_mode = WAL");
|
||||
db.run(`
|
||||
CREATE TABLE IF NOT EXISTS jobs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
external_id TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
company TEXT,
|
||||
url TEXT NOT NULL,
|
||||
location TEXT,
|
||||
tags TEXT NOT NULL DEFAULT '[]',
|
||||
posted_at TEXT,
|
||||
first_seen_at TEXT NOT NULL, -- when the scraper first found it
|
||||
last_seen_at TEXT NOT NULL, -- last scrape that still listed it
|
||||
seen_at TEXT, -- when you marked it as seen (NULL = new)
|
||||
UNIQUE (source, external_id)
|
||||
)
|
||||
`);
|
||||
db.run(`
|
||||
CREATE TABLE IF NOT EXISTS scrape_runs (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
started_at TEXT NOT NULL,
|
||||
finished_at TEXT,
|
||||
found INTEGER,
|
||||
new_count INTEGER,
|
||||
error TEXT
|
||||
)
|
||||
`);
|
||||
|
||||
export interface JobRow {
|
||||
id: number;
|
||||
source: string;
|
||||
external_id: string;
|
||||
title: string;
|
||||
company: string | null;
|
||||
url: string;
|
||||
location: string | null;
|
||||
tags: string;
|
||||
posted_at: string | null;
|
||||
first_seen_at: string;
|
||||
last_seen_at: string;
|
||||
seen_at: string | null;
|
||||
/** 0 if the job was missing from the source's latest successful scrape. */
|
||||
still_listed: number;
|
||||
}
|
||||
|
||||
export interface RunRow {
|
||||
id: number;
|
||||
source: string;
|
||||
started_at: string;
|
||||
finished_at: string | null;
|
||||
found: number | null;
|
||||
new_count: number | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
const upsertJob = db.prepare(`
|
||||
INSERT INTO jobs (source, external_id, title, company, url, location, tags, posted_at, first_seen_at, last_seen_at)
|
||||
VALUES ($source, $externalId, $title, $company, $url, $location, $tags, $postedAt, $now, $now)
|
||||
ON CONFLICT (source, external_id) DO UPDATE SET
|
||||
title = excluded.title, company = excluded.company, url = excluded.url, location = excluded.location,
|
||||
tags = excluded.tags, posted_at = excluded.posted_at, last_seen_at = excluded.last_seen_at
|
||||
RETURNING first_seen_at = $now AS is_new
|
||||
`);
|
||||
|
||||
/** Insert or refresh scraped jobs; returns how many were not in the DB before. */
|
||||
export const saveJobs = db.transaction((source: string, jobs: ScrapedJob[], now: string): number => {
|
||||
let newCount = 0;
|
||||
for (const j of jobs) {
|
||||
const row = upsertJob.get({
|
||||
source,
|
||||
externalId: j.externalId,
|
||||
title: j.title,
|
||||
company: j.company ?? null,
|
||||
url: j.url,
|
||||
location: j.location ?? null,
|
||||
tags: JSON.stringify(j.tags ?? []),
|
||||
postedAt: j.postedAt ?? null,
|
||||
now,
|
||||
}) as { is_new: number };
|
||||
newCount += row.is_new;
|
||||
}
|
||||
return newCount;
|
||||
});
|
||||
|
||||
export function startRun(source: string, now: string): number {
|
||||
const row = db
|
||||
.query("INSERT INTO scrape_runs (source, started_at) VALUES (?, ?) RETURNING id")
|
||||
.get(source, now) as { id: number };
|
||||
return row.id;
|
||||
}
|
||||
|
||||
export function finishRun(id: number, result: { found?: number; newCount?: number; error?: string }) {
|
||||
db.query("UPDATE scrape_runs SET finished_at = ?, found = ?, new_count = ?, error = ? WHERE id = ?").run(
|
||||
new Date().toISOString(),
|
||||
result.found ?? null,
|
||||
result.newCount ?? null,
|
||||
result.error ?? null,
|
||||
id,
|
||||
);
|
||||
}
|
||||
|
||||
export type Filter = "new" | "seen" | "all";
|
||||
|
||||
export function listJobs(opts: { filter: Filter; source?: string; q?: string }): JobRow[] {
|
||||
const where: string[] = [];
|
||||
const params: Record<string, string> = {};
|
||||
if (opts.filter === "new") where.push("j.seen_at IS NULL");
|
||||
if (opts.filter === "seen") where.push("j.seen_at IS NOT NULL");
|
||||
if (opts.source) {
|
||||
where.push("j.source = $source");
|
||||
params.source = opts.source;
|
||||
}
|
||||
if (opts.q) {
|
||||
where.push("(j.title LIKE $q OR j.company LIKE $q OR j.tags LIKE $q)");
|
||||
params.q = `%${opts.q}%`;
|
||||
}
|
||||
return db
|
||||
.query(
|
||||
`SELECT j.*,
|
||||
j.last_seen_at >= COALESCE(
|
||||
(SELECT MAX(r.started_at) FROM scrape_runs r WHERE r.source = j.source AND r.error IS NULL AND r.finished_at IS NOT NULL),
|
||||
j.last_seen_at
|
||||
) AS still_listed
|
||||
FROM jobs j
|
||||
${where.length ? "WHERE " + where.join(" AND ") : ""}
|
||||
ORDER BY still_listed DESC, COALESCE(j.posted_at, j.first_seen_at) DESC`,
|
||||
)
|
||||
.all(params) as JobRow[];
|
||||
}
|
||||
|
||||
export function counts(): { new: number; seen: number; all: number } {
|
||||
return db
|
||||
.query(
|
||||
`SELECT SUM(seen_at IS NULL) AS new, SUM(seen_at IS NOT NULL) AS seen, COUNT(*) AS "all" FROM jobs`,
|
||||
)
|
||||
.get() as { new: number; seen: number; all: number };
|
||||
}
|
||||
|
||||
export function setSeen(id: number, seen: boolean) {
|
||||
db.query("UPDATE jobs SET seen_at = ? WHERE id = ?").run(seen ? new Date().toISOString() : null, id);
|
||||
}
|
||||
|
||||
export function markAllSeen(ids: number[]) {
|
||||
const stmt = db.query("UPDATE jobs SET seen_at = ? WHERE id = ? AND seen_at IS NULL");
|
||||
const now = new Date().toISOString();
|
||||
db.transaction(() => ids.forEach((id) => stmt.run(now, id)))();
|
||||
}
|
||||
|
||||
export function sources(): string[] {
|
||||
return (db.query("SELECT DISTINCT source FROM jobs ORDER BY source").all() as { source: string }[]).map(
|
||||
(r) => r.source,
|
||||
);
|
||||
}
|
||||
|
||||
export function recentRuns(limit = 10): RunRow[] {
|
||||
return db.query("SELECT * FROM scrape_runs ORDER BY id DESC LIMIT ?").all(limit) as RunRow[];
|
||||
}
|
||||
|
||||
/** Start time of the most recent run that finished (successfully or not), across all sources. */
|
||||
export function lastRunAt(): string | null {
|
||||
const row = db.query("SELECT MAX(started_at) AS t FROM scrape_runs WHERE finished_at IS NOT NULL").get() as {
|
||||
t: string | null;
|
||||
};
|
||||
return row.t;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
// Entry point (also used by the compiled binary):
|
||||
// job-board [serve] [--port <n>] start the web server (scrapes every 24h)
|
||||
// job-board scrape run all scrapers once and exit (exit code 1 if any failed)
|
||||
|
||||
export {};
|
||||
|
||||
const USAGE = "Usage: job-board [serve] [--port <n>] | job-board scrape";
|
||||
|
||||
function fail(message: string): never {
|
||||
console.error(`${message}\n${USAGE}`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
let command: string | undefined;
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]!;
|
||||
if (arg === "--port" || arg === "-p" || arg.startsWith("--port=")) {
|
||||
const value = arg.startsWith("--port=") ? arg.slice("--port=".length) : args[++i];
|
||||
const port = Number(value);
|
||||
if (!Number.isInteger(port) || port < 1 || port > 65535) fail(`Invalid port "${value ?? ""}".`);
|
||||
process.env.PORT = String(port); // read by server.ts; overrides the PORT env var
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
console.log(USAGE);
|
||||
process.exit(0);
|
||||
} else if (!command && !arg.startsWith("-")) {
|
||||
command = arg;
|
||||
} else {
|
||||
fail(`Unknown argument "${arg}".`);
|
||||
}
|
||||
}
|
||||
|
||||
if (command === "scrape") {
|
||||
const { runAllScrapers } = await import("./scrape");
|
||||
const results = await runAllScrapers();
|
||||
process.exit(results.some((r) => r.error) ? 1 : 0);
|
||||
} else if (command === undefined || command === "serve") {
|
||||
await import("./server");
|
||||
} else {
|
||||
fail(`Unknown command "${command}".`);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { finishRun, saveJobs, startRun } from "./db";
|
||||
import { scrapers } from "./scrapers";
|
||||
|
||||
export interface RunSummary {
|
||||
source: string;
|
||||
found?: number;
|
||||
newCount?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
let running: Promise<RunSummary[]> | null = null;
|
||||
|
||||
/** Run every registered scraper. A failing scraper is logged and doesn't stop the others. */
|
||||
export function runAllScrapers(): Promise<RunSummary[]> {
|
||||
running ??= (async () => {
|
||||
const results: RunSummary[] = [];
|
||||
for (const s of scrapers) {
|
||||
const now = new Date().toISOString();
|
||||
const runId = startRun(s.id, now);
|
||||
try {
|
||||
const jobs = await s.scrape();
|
||||
if (jobs.length === 0) throw new Error("scraper returned 0 jobs (site layout changed?)");
|
||||
const newCount = saveJobs(s.id, jobs, now);
|
||||
finishRun(runId, { found: jobs.length, newCount });
|
||||
results.push({ source: s.id, found: jobs.length, newCount });
|
||||
console.log(`[scrape] ${s.id}: ${jobs.length} found, ${newCount} new`);
|
||||
} catch (e) {
|
||||
const error = e instanceof Error ? e.message : String(e);
|
||||
finishRun(runId, { error });
|
||||
results.push({ source: s.id, error });
|
||||
console.error(`[scrape] ${s.id} failed: ${error}`);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
})().finally(() => (running = null));
|
||||
return running;
|
||||
}
|
||||
|
||||
export const isScraping = () => running !== null;
|
||||
|
||||
if (import.meta.main) {
|
||||
const results = await runAllScrapers();
|
||||
process.exit(results.some((r) => r.error) ? 1 : 0);
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import { slugify, USER_AGENT } from "./http";
|
||||
import type { ScrapedJob, Scraper } from "./types";
|
||||
|
||||
// Adecco has no public API, but its job search page loads results from this internal JSON endpoint
|
||||
// (a Solr-style query passed as `queryString`). Unknown filter fields are silently ignored by the
|
||||
// API, so results are checked against the requested category.
|
||||
const API_URL = "https://www.adecco.com/api/data/jobs/summarized";
|
||||
const JOB_PAGE = "https://www.adecco.com/sl-si/prosta-delovna-mesta";
|
||||
const MAX_PAGES = 50; // 10 jobs per page
|
||||
|
||||
interface AdeccoJob {
|
||||
jobId: string;
|
||||
jobTitle: string;
|
||||
jobLocation: string | null;
|
||||
cityName: string | null;
|
||||
postedDate: string | null;
|
||||
jobCategoryId: string | null;
|
||||
jobSubCategoryId: string | null;
|
||||
contractTypeTitle: string | null;
|
||||
employmentTypeTitle: string | null;
|
||||
industryTypeTitle: string | null;
|
||||
}
|
||||
|
||||
interface AdeccoResponse {
|
||||
jobs?: AdeccoJob[];
|
||||
pagination?: { total: number; nextRange: number };
|
||||
}
|
||||
|
||||
/** The site builds "<title>-<location>/<jobid>" URLs; only the job ID matters, the slug is cosmetic. */
|
||||
function jobUrl(j: AdeccoJob): string {
|
||||
const slug = slugify(`${j.jobTitle}-${j.jobLocation ?? ""}`);
|
||||
return `${JOB_PAGE}/${slug}/${j.jobId.toLowerCase()}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scraper for adecco.com (Slovenia), filtered by job category.
|
||||
* `category` is the CategoryFilter value from the site's URL, e.g. "ADCSICAT014" (IT).
|
||||
*/
|
||||
export function adecco(category: string, label: string): Scraper {
|
||||
return {
|
||||
id: `adecco-si-${category.toLowerCase()}`,
|
||||
name: `Adecco (${label})`,
|
||||
async scrape() {
|
||||
const raw: AdeccoJob[] = [];
|
||||
for (let page = 0, range = 0; page < MAX_PAGES; page++) {
|
||||
const res = await fetch(API_URL, {
|
||||
method: "POST",
|
||||
headers: { "User-Agent": USER_AGENT, "content-type": "text/plain;charset=UTF-8" },
|
||||
body: JSON.stringify({
|
||||
queryString: `&fq=JobCategoryId:${category}`,
|
||||
range,
|
||||
siteName: "adecco",
|
||||
brand: "adecco",
|
||||
countryCode: "SI",
|
||||
languageCode: "sl-SI",
|
||||
}),
|
||||
});
|
||||
if (!res.ok) throw new Error(`POST ${API_URL} -> HTTP ${res.status}`);
|
||||
const data = (await res.json()) as AdeccoResponse;
|
||||
if (!Array.isArray(data.jobs)) throw new Error("unexpected API response (no `jobs` array)");
|
||||
raw.push(...data.jobs);
|
||||
const total = data.pagination?.total ?? 0;
|
||||
if (data.jobs.length === 0 || raw.length >= total || !data.pagination?.nextRange) break;
|
||||
range = data.pagination.nextRange;
|
||||
}
|
||||
|
||||
const jobs = raw.filter((j) => j.jobCategoryId === category || j.jobSubCategoryId === category);
|
||||
if (raw.length > 0 && jobs.length === 0) {
|
||||
throw new Error(`category filter ${category} was ignored by the API (query format changed?)`);
|
||||
}
|
||||
|
||||
return jobs.map((j) => ({
|
||||
externalId: j.jobId,
|
||||
title: j.jobTitle.trim(),
|
||||
url: jobUrl(j),
|
||||
location: j.cityName || j.jobLocation || undefined,
|
||||
tags: [j.contractTypeTitle, j.employmentTypeTitle, j.industryTypeTitle].filter(
|
||||
(t): t is string => !!t?.trim(),
|
||||
),
|
||||
postedAt: j.postedDate ?? undefined,
|
||||
}));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// Shared helpers for scrapers.
|
||||
|
||||
export const USER_AGENT =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0 Safari/537.36";
|
||||
|
||||
// Upper half (0xA0-0xFF) of ISO-8859-2, which Bun's TextDecoder does not support.
|
||||
const LATIN2_HIGH =
|
||||
" Ą˘Ł¤ĽŚ§¨ŠŞŤŹŽŻ°ą˛ł´ľśˇ¸šşťź˝žżŔÁÂĂÄĹĆÇČÉĘËĚÍÎĎĐŃŇÓÔŐÖ×ŘŮÚŰÜÝŢßŕáâăäĺćçčéęëěíîďđńňóôőö÷řůúűüýţ˙";
|
||||
|
||||
function decodeLatin2(bytes: Uint8Array): string {
|
||||
let out = "";
|
||||
for (const b of bytes) out += b < 0xa0 ? String.fromCharCode(b) : LATIN2_HIGH[b - 0xa0];
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Fetch a page and decode it according to its declared charset. */
|
||||
export async function fetchHtml(url: string): Promise<string> {
|
||||
const res = await fetch(url, { headers: { "User-Agent": USER_AGENT } });
|
||||
if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`);
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
const charset = /charset=([\w-]+)/i.exec(res.headers.get("content-type") ?? "")?.[1]?.toLowerCase() ?? "utf-8";
|
||||
if (charset === "iso-8859-2" || charset === "latin2") return decodeLatin2(bytes);
|
||||
try {
|
||||
return new TextDecoder(charset).decode(bytes);
|
||||
} catch {
|
||||
return new TextDecoder("utf-8").decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
const NAMED_ENTITIES: Record<string, string> = {
|
||||
amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", raquo: "»", laquo: "«",
|
||||
scaron: "š", Scaron: "Š", zcaron: "ž", Zcaron: "Ž", ccaron: "č", Ccaron: "Č",
|
||||
ndash: "–", mdash: "—", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", bdquo: "„", hellip: "…",
|
||||
};
|
||||
|
||||
/** Decode HTML entities (HTMLRewriter text chunks are not decoded). */
|
||||
export function decodeEntities(s: string): string {
|
||||
return s.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (m, e: string) => {
|
||||
if (e[0] === "#") {
|
||||
const code = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
|
||||
return Number.isFinite(code) ? String.fromCodePoint(code) : m;
|
||||
}
|
||||
return NAMED_ENTITIES[e] ?? NAMED_ENTITIES[e.toLowerCase()] ?? m;
|
||||
});
|
||||
}
|
||||
|
||||
export const clean = (s: string) => decodeEntities(s).replace(/\s+/g, " ").trim();
|
||||
|
||||
/** ASCII URL/ID slug: "Inženir (m/ž)" -> "inzenir-m-z". */
|
||||
export const slugify = (s: string) =>
|
||||
s
|
||||
.normalize("NFD")
|
||||
.replace(/[̀-ͯ]/g, "")
|
||||
.replace(/[^A-Za-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "")
|
||||
.toLowerCase();
|
||||
@@ -0,0 +1,24 @@
|
||||
import { adecco } from "./adecco";
|
||||
import { mojeDelo } from "./mojedelo";
|
||||
import { optius } from "./optius";
|
||||
import { sloTech } from "./slotech";
|
||||
import { startupSi } from "./startupsi";
|
||||
import type { Scraper } from "./types";
|
||||
|
||||
/**
|
||||
* All active scrapers. To add a site:
|
||||
* 1. create src/scrapers/<site>.ts exporting an object implementing `Scraper`
|
||||
* 2. add it to this list
|
||||
*/
|
||||
export const scrapers: Scraper[] = [
|
||||
sloTech("Ljubljana"),
|
||||
startupSi(),
|
||||
adecco("ADCSICAT014", "IT"),
|
||||
mojeDelo(
|
||||
"IT, Osrednjeslovenska",
|
||||
"https://www.mojedelo.com/iskanje-zaposlitve?jobCategoryIds=64f003ff-6d8b-4be0-b58c-4580e4eeeb8a®ionIds=d1dce9b1-9fa4-438b-b582-10d371d442e6",
|
||||
),
|
||||
optius("IT, Osrednjeslovenska", "https://www.optius.com/rss/?regija=7&podrocje=7"),
|
||||
];
|
||||
|
||||
export type { Scraper, ScrapedJob } from "./types";
|
||||
@@ -0,0 +1,100 @@
|
||||
import { slugify, USER_AGENT } from "./http";
|
||||
import type { ScrapedJob, Scraper } from "./types";
|
||||
|
||||
// MojeDelo is a single-page app with no public API; its search page calls this internal JSON API.
|
||||
// The API requires tenant headers, which the site itself reads from a public config file.
|
||||
const API = "https://api.mojedelo.com";
|
||||
const CONFIG_URL = `${API}/uploaded-files/config/www.mojedelo.com/jb.globals.js`;
|
||||
const PAGE_SIZE = 20;
|
||||
const MAX_PAGES = 25;
|
||||
|
||||
interface Named {
|
||||
id: string;
|
||||
translation?: string;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
interface MojeDeloJob {
|
||||
id: string;
|
||||
title: string;
|
||||
startDate: string | null;
|
||||
repostingDate: string | null;
|
||||
company: { name?: string } | null;
|
||||
anonymizedCompany: { name?: string } | null;
|
||||
town: Named | null;
|
||||
regions: Named[] | null;
|
||||
jobCategories: Named[] | null;
|
||||
workModes: Named[] | null;
|
||||
}
|
||||
|
||||
interface SearchResponse {
|
||||
data?: { items?: MojeDeloJob[]; total?: number };
|
||||
}
|
||||
|
||||
async function tenantHeaders(): Promise<Record<string, string>> {
|
||||
const res = await fetch(CONFIG_URL, { headers: { "User-Agent": USER_AGENT } });
|
||||
if (!res.ok) throw new Error(`GET ${CONFIG_URL} -> HTTP ${res.status}`);
|
||||
const config = await res.text();
|
||||
const get = (key: string) => {
|
||||
const value = new RegExp(`["']?${key}["']?\\s*:\\s*["']([^"']+)["']`).exec(config)?.[1];
|
||||
if (!value) throw new Error(`${key} not found in site config (site changed?)`);
|
||||
return value;
|
||||
};
|
||||
return { tenantId: get("tenantId"), channelId: get("jbChannelId"), languageId: get("defaultLanguageId") };
|
||||
}
|
||||
|
||||
/**
|
||||
* Scraper for mojedelo.com. Pass a search URL copied from the site (filters in the query string,
|
||||
* e.g. jobCategoryIds / regionIds). `label` is shown in the UI and also forms the stable source ID,
|
||||
* so don't change it later.
|
||||
*/
|
||||
export function mojeDelo(label: string, searchUrl: string): Scraper {
|
||||
const filters = new URL(searchUrl).searchParams;
|
||||
filters.delete("page");
|
||||
|
||||
return {
|
||||
id: `mojedelo-${slugify(label)}`,
|
||||
name: `MojeDelo (${label})`,
|
||||
async scrape() {
|
||||
const headers = { "User-Agent": USER_AGENT, ...(await tenantHeaders()) };
|
||||
const raw: MojeDeloJob[] = [];
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
const params = new URLSearchParams(filters);
|
||||
params.set("pageSize", String(PAGE_SIZE));
|
||||
params.set("startFrom", String(page * PAGE_SIZE));
|
||||
const url = `${API}/job-ads-search?${params}`;
|
||||
const res = await fetch(url, { headers });
|
||||
if (!res.ok) throw new Error(`GET ${url} -> HTTP ${res.status}`);
|
||||
const body = (await res.json()) as SearchResponse;
|
||||
const items = body.data?.items;
|
||||
if (!Array.isArray(items)) throw new Error("unexpected API response (no `data.items` array)");
|
||||
raw.push(...items);
|
||||
if (items.length < PAGE_SIZE || raw.length >= (body.data?.total ?? 0)) break;
|
||||
}
|
||||
|
||||
// The API may silently ignore filters it doesn't recognise, so check results actually match.
|
||||
const wanted = (param: string) => new Set(filters.getAll(param).flatMap((v) => v.split(",")));
|
||||
const categories = wanted("jobCategoryIds");
|
||||
const regions = wanted("regionIds");
|
||||
const matches = (list: Named[] | null, ids: Set<string>) => ids.size === 0 || !!list?.some((x) => ids.has(x.id));
|
||||
const jobs = raw.filter((j) => matches(j.jobCategories, categories) && matches(j.regions, regions));
|
||||
if (raw.length > 0 && jobs.length === 0) {
|
||||
throw new Error("search filters were ignored by the API (query format changed?)");
|
||||
}
|
||||
|
||||
return jobs.map(
|
||||
(j): ScrapedJob => ({
|
||||
externalId: j.id,
|
||||
title: j.title.trim(),
|
||||
url: `https://www.mojedelo.com/oglas/${slugify(j.title)}/${j.id}`,
|
||||
company: j.anonymizedCompany?.name ?? j.company?.name ?? undefined,
|
||||
location: j.town?.translation ?? j.town?.name ?? undefined,
|
||||
tags: [...(j.workModes ?? []), ...(j.jobCategories ?? [])]
|
||||
.map((x) => x.translation?.trim())
|
||||
.filter((t): t is string => !!t),
|
||||
postedAt: j.repostingDate ?? j.startDate ?? undefined,
|
||||
}),
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
import { clean, fetchHtml, slugify } from "./http";
|
||||
import type { ScrapedJob, Scraper } from "./types";
|
||||
|
||||
// Optius publishes an official RSS feed with region/field filters, so no HTML scraping is needed.
|
||||
// Build a feed URL at https://www.optius.com/rss/ (its IDs differ from the search page's IDs).
|
||||
|
||||
const tag = (xml: string, name: string) => new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`).exec(xml)?.[1];
|
||||
const text = (s: string | undefined) => clean((s ?? "").replace(/^<!\[CDATA\[([\s\S]*)\]\]>$/, "$1"));
|
||||
|
||||
export function parseOptiusFeed(xml: string): ScrapedJob[] {
|
||||
return [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)]
|
||||
.map((m): ScrapedJob | null => {
|
||||
const item = m[1]!;
|
||||
const url = text(tag(item, "link") ?? tag(item, "guid"));
|
||||
const id = /-(\d+)-\1\/?$/.exec(url)?.[1] ?? /-(\d+)\/?$/.exec(url)?.[1];
|
||||
const title = text(tag(item, "title"));
|
||||
if (!id || !title) return null;
|
||||
const pubDate = tag(item, "pubDate");
|
||||
return {
|
||||
externalId: id,
|
||||
title,
|
||||
url,
|
||||
company: text(tag(item, "employer")) || undefined,
|
||||
tags: [...item.matchAll(/<field(?:\s[^>]*)?>([\s\S]*?)<\/field>/g)].map((f) => text(f[1])).filter(Boolean),
|
||||
postedAt: pubDate ? new Date(pubDate).toISOString() : undefined,
|
||||
};
|
||||
})
|
||||
.filter((j): j is ScrapedJob => j !== null);
|
||||
}
|
||||
|
||||
/** Scraper for an optius.com RSS feed, e.g. "https://www.optius.com/rss/?regija=7&podrocje=7". */
|
||||
export function optius(label: string, feedUrl: string): Scraper {
|
||||
return {
|
||||
id: `optius-${label.normalize("NFD").replace(/[̀-ͯ]/g, "").replace(/[^A-Za-z0-9]+/g, "-").replace(/^-|-$/g, "").toLowerCase()}`,
|
||||
name: `Optius (${label})`,
|
||||
async scrape() {
|
||||
const xml = await fetchHtml(feedUrl);
|
||||
if (!xml.includes("<rss")) throw new Error("response is not an RSS feed (site changed?)");
|
||||
return parseOptiusFeed(xml);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { clean, fetchHtml } from "./http";
|
||||
import type { ScrapedJob, Scraper } from "./types";
|
||||
|
||||
const BASE = "https://slo-tech.com";
|
||||
|
||||
/** Scraper for slo-tech.com job listings, filtered by city (e.g. "Ljubljana", "Maribor"). */
|
||||
export function sloTech(city: string): Scraper {
|
||||
return {
|
||||
id: `slo-tech-${city.toLowerCase()}`,
|
||||
name: `Slo-Tech (${city})`,
|
||||
async scrape() {
|
||||
const html = await fetchHtml(`${BASE}/delo/mesto/${encodeURIComponent(city)}`);
|
||||
return parseSloTech(html, city);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function parseSloTech(html: string, city: string): Promise<ScrapedJob[]> {
|
||||
type Draft = { href: string; title: string; company: string; tags: string[]; postedAt?: string };
|
||||
const drafts: Draft[] = [];
|
||||
let cur: Draft | undefined;
|
||||
|
||||
await new HTMLRewriter()
|
||||
.on("table.forums tbody tr", {
|
||||
element() {
|
||||
cur = { href: "", title: "", company: "", tags: [] };
|
||||
drafts.push(cur);
|
||||
},
|
||||
})
|
||||
.on("table.forums tbody td.name h3 a", {
|
||||
element(el) {
|
||||
if (cur) cur.href = el.getAttribute("href") ?? "";
|
||||
},
|
||||
text(t) {
|
||||
if (cur) cur.title += t.text;
|
||||
},
|
||||
})
|
||||
.on("table.forums tbody td.name span.oddelek a", {
|
||||
element() {
|
||||
cur?.tags.push("");
|
||||
},
|
||||
text(t) {
|
||||
if (cur?.tags.length) cur.tags[cur.tags.length - 1] += t.text;
|
||||
},
|
||||
})
|
||||
.on("table.forums tbody td.company", {
|
||||
text(t) {
|
||||
if (cur) cur.company += t.text;
|
||||
},
|
||||
})
|
||||
.on("table.forums tbody td.last_msg time", {
|
||||
element(el) {
|
||||
if (cur) cur.postedAt = el.getAttribute("datetime") ?? undefined;
|
||||
},
|
||||
})
|
||||
.transform(new Response(html))
|
||||
.text();
|
||||
|
||||
return drafts
|
||||
.map((d): ScrapedJob | null => {
|
||||
const id = /\/delo\/(\d+)/.exec(d.href)?.[1];
|
||||
if (!id) return null;
|
||||
return {
|
||||
externalId: id,
|
||||
title: clean(d.title),
|
||||
url: new URL(d.href, BASE).href,
|
||||
company: clean(d.company) || undefined,
|
||||
location: city,
|
||||
tags: d.tags.map(clean).filter(Boolean),
|
||||
postedAt: d.postedAt ? new Date(d.postedAt).toISOString() : undefined,
|
||||
};
|
||||
})
|
||||
.filter((j): j is ScrapedJob => j !== null);
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import { clean, fetchHtml } from "./http";
|
||||
import type { ScrapedJob, Scraper } from "./types";
|
||||
|
||||
// startup.si has no public jobs API or feed, so we parse the listing page. It holds every posting
|
||||
// on a single page (no pagination), going back years, so old postings are filtered out by age.
|
||||
const LIST_URL = "https://www.startup.si/sl-si/za-startupe/oglasi/zaposlitve";
|
||||
|
||||
const MONTHS = ["januar", "februar", "marec", "april", "maj", "junij", "julij", "avgust", "september", "oktober", "november", "december"];
|
||||
|
||||
/** "21. september 2026" -> ISO date */
|
||||
function parseSlDate(s: string): string | undefined {
|
||||
const m = /(\d{1,2})\.\s*([a-zčšž]+)\s+(\d{4})/i.exec(s);
|
||||
const month = m ? MONTHS.indexOf(m[2]!.toLowerCase()) : -1;
|
||||
if (!m || month < 0) return undefined;
|
||||
return new Date(Date.UTC(Number(m[3]), month, Number(m[1]))).toISOString();
|
||||
}
|
||||
|
||||
/** Links on this site are hand-entered: some are empty, relative, or missing the scheme. */
|
||||
function normalizeUrl(href: string): string {
|
||||
const h = clean(href);
|
||||
if (!h) return LIST_URL;
|
||||
if (/^www\./i.test(h)) return `https://${h}`;
|
||||
try {
|
||||
return new URL(h, LIST_URL + "/").href;
|
||||
} catch {
|
||||
return LIST_URL;
|
||||
}
|
||||
}
|
||||
|
||||
/** Scraper for startup.si job ads. Only postings from the last `maxAgeDays` days are returned. */
|
||||
export function startupSi(maxAgeDays = 90): Scraper {
|
||||
return {
|
||||
id: "startup-si",
|
||||
name: "Startup.si",
|
||||
async scrape() {
|
||||
const jobs = await parseStartupSi(await fetchHtml(LIST_URL));
|
||||
if (jobs.length === 0) throw new Error("no job cards found (site layout changed?)");
|
||||
const cutoff = Date.now() - maxAgeDays * 86400_000;
|
||||
return jobs.filter((j) => !j.postedAt || new Date(j.postedAt).getTime() >= cutoff);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function parseStartupSi(html: string): Promise<ScrapedJob[]> {
|
||||
type Draft = { id: string; title: string; date: string; href?: string };
|
||||
const drafts: Draft[] = [];
|
||||
let cur: Draft | undefined;
|
||||
let lastHref = "";
|
||||
|
||||
await new HTMLRewriter()
|
||||
.on(".advertisment-card", {
|
||||
element() {
|
||||
cur = { id: "", title: "", date: "" };
|
||||
drafts.push(cur);
|
||||
},
|
||||
})
|
||||
.on(".advertisment-card .right > .title", {
|
||||
text(t) {
|
||||
if (cur) cur.title += t.text;
|
||||
},
|
||||
})
|
||||
.on(".advertisment-card .title button", {
|
||||
element(el) {
|
||||
if (cur) cur.id = (el.getAttribute("data-target") ?? "").replace(/^#description/, "");
|
||||
},
|
||||
})
|
||||
.on(".advertisment-card .published-info b", {
|
||||
text(t) {
|
||||
if (cur) cur.date += t.text;
|
||||
},
|
||||
})
|
||||
// The "preberi več" (read more) link is an <a> wrapping span.read-more; descriptions can contain other links.
|
||||
.on(".advertisment-card a", {
|
||||
element(el) {
|
||||
lastHref = el.getAttribute("href") ?? "";
|
||||
},
|
||||
})
|
||||
.on(".advertisment-card span.read-more", {
|
||||
element() {
|
||||
if (cur) cur.href = lastHref;
|
||||
},
|
||||
})
|
||||
.transform(new Response(html))
|
||||
.text();
|
||||
|
||||
return drafts
|
||||
.filter((d) => d.id && clean(d.title))
|
||||
.map((d) => ({
|
||||
externalId: d.id,
|
||||
title: clean(d.title),
|
||||
url: normalizeUrl(d.href ?? ""),
|
||||
postedAt: parseSlDate(clean(d.date)),
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
/** A single job posting as returned by a scraper. */
|
||||
export interface ScrapedJob {
|
||||
/** Stable ID unique within the source (used to detect new vs. already-known jobs). */
|
||||
externalId: string;
|
||||
title: string;
|
||||
url: string;
|
||||
company?: string;
|
||||
location?: string;
|
||||
tags?: string[];
|
||||
/** ISO 8601 date the job was posted, if the site exposes it. */
|
||||
postedAt?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Implement this interface to add a new site.
|
||||
* Then register the instance in `src/scrapers/index.ts`.
|
||||
*/
|
||||
export interface Scraper {
|
||||
/** Unique, stable identifier stored in the DB (e.g. "slo-tech-ljubljana"). Don't change it later. */
|
||||
id: string;
|
||||
/** Human-readable name shown in the UI. */
|
||||
name: string;
|
||||
scrape(): Promise<ScrapedJob[]>;
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
import { counts, lastRunAt, listJobs, markAllSeen, recentRuns, setSeen, sources, type Filter, type JobRow } from "./db";
|
||||
import { isScraping, runAllScrapers } from "./scrape";
|
||||
import { scrapers } from "./scrapers";
|
||||
|
||||
const PORT = Number(process.env.PORT ?? 3000);
|
||||
const SCRAPE_INTERVAL_HOURS = Number(process.env.SCRAPE_INTERVAL_HOURS ?? 24);
|
||||
|
||||
// --- Scheduler: scrape every SCRAPE_INTERVAL_HOURS. On startup, wait out whatever is left of the
|
||||
// interval since the last run (or scrape right away if it's overdue). ---
|
||||
const INTERVAL_MS = SCRAPE_INTERVAL_HOURS * 3600_000;
|
||||
async function scheduledScrape() {
|
||||
await runAllScrapers();
|
||||
setTimeout(scheduledScrape, INTERVAL_MS);
|
||||
}
|
||||
const last = lastRunAt();
|
||||
setTimeout(scheduledScrape, last ? Math.max(0, new Date(last).getTime() + INTERVAL_MS - Date.now()) : 0);
|
||||
|
||||
// --- HTML rendering ---
|
||||
const esc = (s: unknown) =>
|
||||
String(s ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c]!);
|
||||
|
||||
const fmtDate = (iso: string | null) =>
|
||||
iso ? new Date(iso).toLocaleDateString("sl-SI", { day: "numeric", month: "short", year: "numeric" }) : "";
|
||||
const fmtDateTime = (iso: string | null) =>
|
||||
iso ? new Date(iso).toLocaleString("sl-SI", { dateStyle: "medium", timeStyle: "short" }) : "never";
|
||||
|
||||
const sourceName = (id: string) => scrapers.find((s) => s.id === id)?.name ?? id;
|
||||
|
||||
function jobCard(j: JobRow) {
|
||||
const tags = JSON.parse(j.tags) as string[];
|
||||
return `
|
||||
<li class="job ${j.seen_at ? "seen" : "new"} ${j.still_listed ? "" : "gone"}" data-id="${j.id}">
|
||||
<div class="main">
|
||||
<div class="title-row">
|
||||
${j.seen_at ? "" : `<span class="badge new-badge">NEW</span>`}
|
||||
${j.still_listed ? "" : `<span class="badge gone-badge" title="Missing from the latest scrape">no longer listed</span>`}
|
||||
<a class="title" href="${esc(j.url)}" target="_blank" rel="noopener">${esc(j.title)}</a>
|
||||
</div>
|
||||
<div class="meta">
|
||||
${j.company ? `<span class="company">${esc(j.company)}</span>` : ""}
|
||||
${j.location ? `<span>${esc(j.location)}</span>` : ""}
|
||||
<span>${esc(sourceName(j.source))}</span>
|
||||
${j.posted_at ? `<span title="Posted">posted ${fmtDate(j.posted_at)}</span>` : ""}
|
||||
<span title="First found by scraper">found ${fmtDate(j.first_seen_at)}</span>
|
||||
</div>
|
||||
${tags.length ? `<div class="tags">${tags.map((t) => `<span class="tag">${esc(t)}</span>`).join("")}</div>` : ""}
|
||||
</div>
|
||||
<button class="toggle" type="button">${j.seen_at ? "Mark new" : "Mark seen"}</button>
|
||||
</li>`;
|
||||
}
|
||||
|
||||
function page(url: URL) {
|
||||
const filter = (["new", "seen", "all"].includes(url.searchParams.get("filter") ?? "")
|
||||
? url.searchParams.get("filter")
|
||||
: "new") as Filter;
|
||||
const source = url.searchParams.get("source") ?? "";
|
||||
const q = url.searchParams.get("q") ?? "";
|
||||
const jobs = listJobs({ filter, source: source || undefined, q: q || undefined });
|
||||
const c = counts();
|
||||
const runs = recentRuns(5);
|
||||
const link = (f: Filter) => {
|
||||
const p = new URLSearchParams(url.searchParams);
|
||||
p.set("filter", f);
|
||||
return `?${p}`;
|
||||
};
|
||||
|
||||
return `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Job Board</title>
|
||||
<style>
|
||||
:root { --bg:#f6f7f9; --card:#fff; --fg:#1c1f24; --muted:#6b7280; --border:#e3e6ea; --accent:#2563eb; --new:#16a34a; --gone:#b45309; --tag:#eef1f5; }
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root { --bg:#111317; --card:#1a1d23; --fg:#e6e8eb; --muted:#9aa1ab; --border:#2a2e36; --accent:#60a5fa; --new:#4ade80; --gone:#fbbf24; --tag:#252a32; }
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin:0; background:var(--bg); color:var(--fg); font:15px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; }
|
||||
.wrap { max-width: 920px; margin: 0 auto; padding: 24px 16px 64px; }
|
||||
header { display:flex; flex-wrap:wrap; align-items:center; gap:12px; justify-content:space-between; margin-bottom:16px; }
|
||||
h1 { font-size:22px; margin:0; }
|
||||
.status { color:var(--muted); font-size:13px; }
|
||||
button, .btn { font:inherit; font-size:13px; padding:6px 12px; border-radius:8px; border:1px solid var(--border); background:var(--card); color:var(--fg); cursor:pointer; }
|
||||
button:hover { border-color:var(--accent); }
|
||||
button:disabled { opacity:.6; cursor:wait; }
|
||||
.toolbar { display:flex; flex-wrap:wrap; gap:8px; align-items:center; margin-bottom:16px; }
|
||||
.tabs { display:flex; gap:4px; background:var(--card); border:1px solid var(--border); border-radius:10px; padding:3px; }
|
||||
.tabs a { padding:5px 12px; border-radius:7px; color:var(--fg); text-decoration:none; font-size:14px; }
|
||||
.tabs a.active { background:var(--accent); color:#fff; }
|
||||
.toolbar form { display:flex; gap:8px; flex:1; min-width:240px; }
|
||||
input, select { font:inherit; font-size:14px; padding:6px 10px; border-radius:8px; border:1px solid var(--border); background:var(--card); color:var(--fg); }
|
||||
input[type=search] { flex:1; min-width:0; }
|
||||
ul.jobs { list-style:none; padding:0; margin:0; display:flex; flex-direction:column; gap:8px; }
|
||||
.job { display:flex; gap:12px; align-items:flex-start; background:var(--card); border:1px solid var(--border); border-left:4px solid var(--new); border-radius:10px; padding:12px 14px; }
|
||||
.job.seen { border-left-color:var(--border); }
|
||||
.job.seen .title { color:var(--muted); }
|
||||
.job.gone { opacity:.65; }
|
||||
.main { flex:1; min-width:0; }
|
||||
.title-row { display:flex; flex-wrap:wrap; align-items:center; gap:6px; }
|
||||
.title { font-weight:600; font-size:16px; color:var(--fg); text-decoration:none; overflow-wrap:anywhere; }
|
||||
.title:hover { color:var(--accent); text-decoration:underline; }
|
||||
.badge { font-size:11px; font-weight:700; padding:1px 6px; border-radius:5px; }
|
||||
.new-badge { color:var(--new); border:1px solid var(--new); }
|
||||
.gone-badge { color:var(--gone); border:1px solid var(--gone); font-weight:500; }
|
||||
.meta { display:flex; flex-wrap:wrap; gap:4px 12px; color:var(--muted); font-size:13px; margin-top:2px; }
|
||||
.company { color:var(--fg); font-weight:500; }
|
||||
.tags { display:flex; flex-wrap:wrap; gap:4px; margin-top:6px; }
|
||||
.tag { background:var(--tag); font-size:12px; padding:1px 8px; border-radius:999px; max-width:100%; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; }
|
||||
.toggle { flex-shrink:0; }
|
||||
.empty { text-align:center; color:var(--muted); padding:48px 0; }
|
||||
details { margin-top:32px; color:var(--muted); font-size:13px; }
|
||||
table { border-collapse:collapse; width:100%; margin-top:8px; }
|
||||
td, th { text-align:left; padding:4px 8px; border-bottom:1px solid var(--border); }
|
||||
.err { color:#dc2626; }
|
||||
@media (max-width: 560px) { .job { flex-direction:column; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<header>
|
||||
<div>
|
||||
<h1>Job Board</h1>
|
||||
<div class="status">Last scrape: ${fmtDateTime(lastRunAt())}${isScraping() ? " · scraping now…" : ""}</div>
|
||||
</div>
|
||||
<button id="scrape" type="button" ${isScraping() ? "disabled" : ""}>Scrape now</button>
|
||||
</header>
|
||||
|
||||
<div class="toolbar">
|
||||
<nav class="tabs">
|
||||
<a href="${link("new")}" class="${filter === "new" ? "active" : ""}">New (<span id="c-new">${c.new ?? 0}</span>)</a>
|
||||
<a href="${link("seen")}" class="${filter === "seen" ? "active" : ""}">Seen (<span id="c-seen">${c.seen ?? 0}</span>)</a>
|
||||
<a href="${link("all")}" class="${filter === "all" ? "active" : ""}">All (${c.all ?? 0})</a>
|
||||
</nav>
|
||||
<form method="get">
|
||||
<input type="hidden" name="filter" value="${filter}">
|
||||
<input type="search" name="q" value="${esc(q)}" placeholder="Search title, company, tags…">
|
||||
<select name="source" onchange="this.form.submit()">
|
||||
<option value="">All sources</option>
|
||||
${sources().map((s) => `<option value="${esc(s)}" ${s === source ? "selected" : ""}>${esc(sourceName(s))}</option>`).join("")}
|
||||
</select>
|
||||
</form>
|
||||
${filter !== "seen" && jobs.some((j) => !j.seen_at) ? `<button id="all-seen" type="button">Mark all shown as seen</button>` : ""}
|
||||
</div>
|
||||
|
||||
${jobs.length ? `<ul class="jobs">${jobs.map(jobCard).join("")}</ul>` : `<p class="empty">${filter === "new" ? "No new jobs. 🎉" : "No jobs."}</p>`}
|
||||
|
||||
<details>
|
||||
<summary>Recent scrape runs</summary>
|
||||
<table>
|
||||
<tr><th>Source</th><th>Started</th><th>Found</th><th>New</th><th>Error</th></tr>
|
||||
${runs.map((r) => `<tr><td>${esc(sourceName(r.source))}</td><td>${fmtDateTime(r.started_at)}</td><td>${r.found ?? ""}</td><td>${r.new_count ?? ""}</td><td class="err">${esc(r.error)}</td></tr>`).join("")}
|
||||
</table>
|
||||
</details>
|
||||
</div>
|
||||
<script>
|
||||
const filter = ${JSON.stringify(filter)};
|
||||
async function setSeen(li, seen) {
|
||||
const res = await fetch("/api/jobs/" + li.dataset.id + "/seen", {
|
||||
method: "POST", headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ seen }), keepalive: true,
|
||||
});
|
||||
const c = await res.json();
|
||||
document.getElementById("c-new").textContent = c.new ?? 0;
|
||||
document.getElementById("c-seen").textContent = c.seen ?? 0;
|
||||
li.classList.toggle("seen", seen);
|
||||
li.classList.toggle("new", !seen);
|
||||
li.querySelector(".new-badge")?.remove();
|
||||
if (!seen) li.querySelector(".title-row").insertAdjacentHTML("afterbegin", '<span class="badge new-badge">NEW</span>');
|
||||
li.querySelector(".toggle").textContent = seen ? "Mark new" : "Mark seen";
|
||||
if ((filter === "new" && seen) || (filter === "seen" && !seen)) li.remove();
|
||||
}
|
||||
document.querySelectorAll(".job").forEach((li) => {
|
||||
li.querySelector(".toggle").addEventListener("click", () => setSeen(li, !li.classList.contains("seen")));
|
||||
// Opening a job counts as seeing it (the card stays until you reload).
|
||||
li.querySelector(".title").addEventListener("click", () => {
|
||||
if (li.classList.contains("seen")) return;
|
||||
fetch("/api/jobs/" + li.dataset.id + "/seen", {
|
||||
method: "POST", headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ seen: true }), keepalive: true,
|
||||
});
|
||||
li.classList.add("seen"); li.classList.remove("new");
|
||||
li.querySelector(".new-badge")?.remove();
|
||||
li.querySelector(".toggle").textContent = "Mark new";
|
||||
});
|
||||
});
|
||||
document.getElementById("all-seen")?.addEventListener("click", async () => {
|
||||
const ids = [...document.querySelectorAll(".job.new")].map((li) => Number(li.dataset.id));
|
||||
await fetch("/api/jobs/seen", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ ids }) });
|
||||
location.reload();
|
||||
});
|
||||
document.getElementById("scrape").addEventListener("click", async (e) => {
|
||||
e.target.disabled = true; e.target.textContent = "Scraping…";
|
||||
const res = await fetch("/api/scrape", { method: "POST" });
|
||||
const results = await res.json();
|
||||
const errors = results.filter((r) => r.error);
|
||||
if (errors.length) alert("Some scrapers failed:\\n" + errors.map((r) => r.source + ": " + r.error).join("\\n"));
|
||||
location.reload();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
const html = (body: string) => new Response(body, { headers: { "content-type": "text/html; charset=utf-8" } });
|
||||
|
||||
Bun.serve({
|
||||
port: PORT,
|
||||
routes: {
|
||||
"/": (req) => html(page(new URL(req.url))),
|
||||
"/api/scrape": { POST: async () => Response.json(await runAllScrapers()) },
|
||||
"/api/jobs/seen": {
|
||||
POST: async (req) => {
|
||||
const { ids } = (await req.json()) as { ids: number[] };
|
||||
markAllSeen(ids.filter(Number.isInteger));
|
||||
return Response.json(counts());
|
||||
},
|
||||
},
|
||||
"/api/jobs/:id/seen": {
|
||||
POST: async (req) => {
|
||||
const { seen } = (await req.json()) as { seen: boolean };
|
||||
setSeen(Number(req.params.id), !!seen);
|
||||
return Response.json(counts());
|
||||
},
|
||||
},
|
||||
},
|
||||
fetch: () => new Response("Not found", { status: 404 }),
|
||||
});
|
||||
|
||||
console.log(`Job board running at http://localhost:${PORT} (scraping every ${SCRAPE_INTERVAL_HOURS}h)`);
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"lib": ["ESNext", "DOM"],
|
||||
"target": "ESNext",
|
||||
"module": "Preserve",
|
||||
"moduleResolution": "bundler",
|
||||
"types": ["bun"],
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"skipLibCheck": true
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user