Add Optius scraper via its official RSS feed
Optius publishes an RSS feed with region/field filters, so no HTML scraping is needed. Also move the duplicated slug helper into http.ts (output unchanged, so existing source IDs and URLs are unaffected). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
import { clean, fetchHtml, slugify } from "./http";
|
||||
import type { ScrapedJob, Scraper } from "./types";
|
||||
|
||||
// Optius publishes an official RSS feed with region/field filters, so no HTML scraping is needed.
|
||||
// Build a feed URL at https://www.optius.com/rss/ (its IDs differ from the search page's IDs).
|
||||
|
||||
const tag = (xml: string, name: string) => new RegExp(`<${name}[^>]*>([\\s\\S]*?)</${name}>`).exec(xml)?.[1];
|
||||
const text = (s: string | undefined) => clean((s ?? "").replace(/^<!\[CDATA\[([\s\S]*)\]\]>$/, "$1"));
|
||||
|
||||
export function parseOptiusFeed(xml: string): ScrapedJob[] {
|
||||
return [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)]
|
||||
.map((m): ScrapedJob | null => {
|
||||
const item = m[1]!;
|
||||
const url = text(tag(item, "link") ?? tag(item, "guid"));
|
||||
const id = /-(\d+)-\1\/?$/.exec(url)?.[1] ?? /-(\d+)\/?$/.exec(url)?.[1];
|
||||
const title = text(tag(item, "title"));
|
||||
if (!id || !title) return null;
|
||||
const pubDate = tag(item, "pubDate");
|
||||
return {
|
||||
externalId: id,
|
||||
title,
|
||||
url,
|
||||
company: text(tag(item, "employer")) || undefined,
|
||||
tags: [...item.matchAll(/<field(?:\s[^>]*)?>([\s\S]*?)<\/field>/g)].map((f) => text(f[1])).filter(Boolean),
|
||||
postedAt: pubDate ? new Date(pubDate).toISOString() : undefined,
|
||||
};
|
||||
})
|
||||
.filter((j): j is ScrapedJob => j !== null);
|
||||
}
|
||||
|
||||
/** Scraper for an optius.com RSS feed, e.g. "https://www.optius.com/rss/?regija=7&podrocje=7". */
|
||||
export function optius(label: string, feedUrl: string): Scraper {
|
||||
return {
|
||||
id: `optius-${label.normalize("NFD").replace(/[̀-ͯ]/g, "").replace(/[^A-Za-z0-9]+/g, "-").replace(/^-|-$/g, "").toLowerCase()}`,
|
||||
name: `Optius (${label})`,
|
||||
async scrape() {
|
||||
const xml = await fetchHtml(feedUrl);
|
||||
if (!xml.includes("<rss")) throw new Error("response is not an RSS feed (site changed?)");
|
||||
return parseOptiusFeed(xml);
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user