nikolaandClaude Opus 5.5
ad2fa2b775
Optius publishes an RSS feed with region/field filters, so no HTML scraping is needed. Also move the duplicated slug helper into http.ts (output unchanged, so existing source IDs and URLs are unaffected). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
43 lines
1.9 KiB
TypeScript
43 lines
1.9 KiB
TypeScript
import { clean, fetchHtml, slugify } from "./http";
|
|
import type { ScrapedJob, Scraper } from "./types";
|
|
|
|
// Optius publishes an official RSS feed with region/field filters, so no HTML scraping is needed.
|
|
// Build a feed URL at https://www.optius.com/rss/ (its IDs differ from the search page's IDs).
|
|
|
|
const tag = (xml: string, name: string) => new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`).exec(xml)?.[1];
|
|
const text = (s: string | undefined) => clean((s ?? "").replace(/^<!\[CDATA\[([\s\S]*)\]\]>$/, "$1"));
|
|
|
|
export function parseOptiusFeed(xml: string): ScrapedJob[] {
|
|
return [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)]
|
|
.map((m): ScrapedJob | null => {
|
|
const item = m[1]!;
|
|
const url = text(tag(item, "link") ?? tag(item, "guid"));
|
|
const id = /-(\d+)-\1\/?$/.exec(url)?.[1] ?? /-(\d+)\/?$/.exec(url)?.[1];
|
|
const title = text(tag(item, "title"));
|
|
if (!id || !title) return null;
|
|
const pubDate = tag(item, "pubDate");
|
|
return {
|
|
externalId: id,
|
|
title,
|
|
url,
|
|
company: text(tag(item, "employer")) || undefined,
|
|
tags: [...item.matchAll(/<field(?:\s[^>]*)?>([\s\S]*?)<\/field>/g)].map((f) => text(f[1])).filter(Boolean),
|
|
postedAt: pubDate ? new Date(pubDate).toISOString() : undefined,
|
|
};
|
|
})
|
|
.filter((j): j is ScrapedJob => j !== null);
|
|
}
|
|
|
|
/** Scraper for an optius.com RSS feed, e.g. "https://www.optius.com/rss/?regija=7&podrocje=7". */
|
|
export function optius(label: string, feedUrl: string): Scraper {
|
|
return {
|
|
id: `optius-${label.normalize("NFD").replace(/[̀-ͯ]/g, "").replace(/[^A-Za-z0-9]+/g, "-").replace(/^-|-$/g, "").toLowerCase()}`,
|
|
name: `Optius (${label})`,
|
|
async scrape() {
|
|
const xml = await fetchHtml(feedUrl);
|
|
if (!xml.includes("<rss")) throw new Error("response is not an RSS feed (site changed?)");
|
|
return parseOptiusFeed(xml);
|
|
},
|
|
};
|
|
}
|