diff --git a/scrapper/.gitignore b/scrapper/.gitignore index ad250eb..df7726a 100644 --- a/scrapper/.gitignore +++ b/scrapper/.gitignore @@ -1,2 +1,3 @@ node_modules data/ +bin/ diff --git a/scrapper/README.md b/scrapper/README.md index 841060c..10b03ed 100644 --- a/scrapper/README.md +++ b/scrapper/README.md @@ -18,13 +18,53 @@ No runtime dependencies. It uses Bun's built-in SQLite, HTTP server and HTMLRewr ```sh bun install -bun start # http://localhost:3000 +bun start # http://localhost:3000 +bun start --port 8080 # or -p 8080 ``` The server scrapes every 24h. On startup it waits out the rest of the 24h since the last scrape (or scrapes right away if none happened in the last 24h). You can also click **Scrape now** in the UI, or run `bun run scrape` once (e.g. from cron). -Env vars: `PORT` (3000), `SCRAPE_INTERVAL_HOURS` (24), `DB_PATH` (`data/jobs.sqlite`). +Env vars: `PORT` (3000; `--port` overrides it), `SCRAPE_INTERVAL_HOURS` (24), `DB_PATH` (`data/jobs.sqlite`). + +## Build a binary for a server + +`bun build --compile` bundles the app and the Bun runtime into one executable (~90 MB), so the +server doesn't need Bun or `node_modules`: + +```sh +bun run build # bin/job-board for this machine +bun run build:linux-x64 # bin/job-board-linux-x64 +bun run build:linux-arm64 # bin/job-board-linux-arm64 (e.g. Raspberry Pi, ARM VPS) +``` + +On the server: + +```sh +./job-board # web server on :3000, scrapes every 24h +./job-board --port 8080 +./job-board scrape # scrape once and exit (exit code 1 if a scraper failed) +``` + +The database goes to `data/jobs.sqlite` next to the binary (override with `DB_PATH`). +The binaries target glibc Linux; for Alpine use `--target=bun-linux-x64-musl`, and for old CPUs +without AVX2 use `--target=bun-linux-x64-baseline`. + +Example systemd unit (`/etc/systemd/system/job-board.service`): + +```ini +[Unit] +Description=Job board +After=network-online.target + +[Service] +ExecStart=/opt/job-board/job-board --port 3000 +Restart=on-failure +User=jobboard + +[Install] +WantedBy=multi-user.target +``` ## UI diff --git a/scrapper/package.json b/scrapper/package.json index 9fc5429..3ad82e6 100644 --- a/scrapper/package.json +++ b/scrapper/package.json @@ -3,9 +3,12 @@ "private": true, "type": "module", "scripts": { - "start": "bun run src/server.ts", - "dev": "bun --watch run src/server.ts", - "scrape": "bun run src/scrape.ts" + "start": "bun run src/main.ts", + "dev": "bun --watch run src/main.ts", + "scrape": "bun run src/scrape.ts", + "build": "bun build src/main.ts --compile --minify --sourcemap --outfile bin/job-board", + "build:linux-x64": "bun build src/main.ts --compile --minify --sourcemap --target=bun-linux-x64 --outfile bin/job-board-linux-x64", + "build:linux-arm64": "bun build src/main.ts --compile --minify --sourcemap --target=bun-linux-arm64 --outfile bin/job-board-linux-arm64" }, "devDependencies": { "@types/bun": "latest" diff --git a/scrapper/src/db.ts b/scrapper/src/db.ts index 96a887b..d2784b1 100644 --- a/scrapper/src/db.ts +++ b/scrapper/src/db.ts @@ -3,7 +3,10 @@ import { mkdirSync } from "node:fs"; import { dirname } from "node:path"; import type { ScrapedJob } from "./scrapers/types"; -const DB_PATH = process.env.DB_PATH ?? `${import.meta.dir}/../data/jobs.sqlite`; +// In a compiled binary (`bun run build`) import.meta.dir is a virtual path, so keep data next to the executable. +const isCompiled = import.meta.dir.startsWith("/$bunfs"); +const baseDir = isCompiled ? dirname(process.execPath) : `${import.meta.dir}/..`; +const DB_PATH = process.env.DB_PATH ?? `${baseDir}/data/jobs.sqlite`; mkdirSync(dirname(DB_PATH), { recursive: true }); export const db = new Database(DB_PATH, { create: true, strict: true }); diff --git a/scrapper/src/main.ts b/scrapper/src/main.ts new file mode 100644 index 0000000..a233682 --- /dev/null +++ b/scrapper/src/main.ts @@ -0,0 +1,41 @@ +// Entry point (also used by the compiled binary): +// job-board [serve] [--port ] start the web server (scrapes every 24h) +// job-board scrape run all scrapers once and exit (exit code 1 if any failed) + +export {}; + +const USAGE = "Usage: job-board [serve] [--port ] | job-board scrape"; + +function fail(message: string): never { + console.error(`${message}\n${USAGE}`); + process.exit(2); +} + +const args = process.argv.slice(2); +let command: string | undefined; +for (let i = 0; i < args.length; i++) { + const arg = args[i]!; + if (arg === "--port" || arg === "-p" || arg.startsWith("--port=")) { + const value = arg.startsWith("--port=") ? arg.slice("--port=".length) : args[++i]; + const port = Number(value); + if (!Number.isInteger(port) || port < 1 || port > 65535) fail(`Invalid port "${value ?? ""}".`); + process.env.PORT = String(port); // read by server.ts; overrides the PORT env var + } else if (arg === "--help" || arg === "-h") { + console.log(USAGE); + process.exit(0); + } else if (!command && !arg.startsWith("-")) { + command = arg; + } else { + fail(`Unknown argument "${arg}".`); + } +} + +if (command === "scrape") { + const { runAllScrapers } = await import("./scrape"); + const results = await runAllScrapers(); + process.exit(results.some((r) => r.error) ? 1 : 0); +} else if (command === undefined || command === "serve") { + await import("./server"); +} else { + fail(`Unknown command "${command}".`); +}