t
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
# dependencies (bun install)
|
||||
node_modules
|
||||
|
||||
# output
|
||||
out
|
||||
dist
|
||||
*.tgz
|
||||
|
||||
# code coverage
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# logs
|
||||
logs
|
||||
_.log
|
||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# caches
|
||||
.eslintcache
|
||||
.cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# IntelliJ based IDEs
|
||||
.idea
|
||||
|
||||
# Finder (MacOS) folder config
|
||||
.DS_Store
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"configVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "job",
|
||||
"dependencies": {
|
||||
"playwright": "^1.59.1",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest",
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@types/bun": ["@types/bun@1.3.12", "", { "dependencies": { "bun-types": "1.3.12" } }, "sha512-DBv81elK+/VSwXHDlnH3Qduw+KxkTIWi7TXkAeh24zpi5l0B2kUg9Ga3tb4nJaPcOFswflgi/yAvMVBPrxMB+A=="],
|
||||
|
||||
"@types/node": ["@types/node@25.6.0", "", { "dependencies": { "undici-types": "~7.19.0" } }, "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ=="],
|
||||
|
||||
"bun-types": ["bun-types@1.3.12", "", { "dependencies": { "@types/node": "*" } }, "sha512-HqOLj5PoFajAQciOMRiIZGNoKxDJSr6qigAttOX40vJuSp6DN/CxWp9s3C1Xwm4oH7ybueITwiaOcWXoYVoRkA=="],
|
||||
|
||||
"fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="],
|
||||
|
||||
"playwright": ["playwright@1.59.1", "", { "dependencies": { "playwright-core": "1.59.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw=="],
|
||||
|
||||
"playwright-core": ["playwright-core@1.59.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"undici-types": ["undici-types@7.19.2", "", {}, "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg=="],
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import playwright from 'playwright';
|
||||
|
||||
function sleep(ms: number) {
|
||||
return Bun.sleep(ms);
|
||||
}
|
||||
|
||||
async function waitUser() {
|
||||
for await (const line of console) {
|
||||
console.log(line);
|
||||
// wait for user enter to continue
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const browser = await playwright.chromium.launch({ headless: false });
|
||||
const context = await browser.newContext();
|
||||
const page = await context.newPage();
|
||||
const page2 = await context.newPage();
|
||||
|
||||
await context.route('**.jpg', route => route.abort());
|
||||
await page.goto('https://www.google.com/maps/search/Software+company/@46.0463511,14.5023435,14z');
|
||||
|
||||
|
||||
await page.click('span:has-text("Sprejmi")');
|
||||
await page.waitForLoadState('networkidle');
|
||||
|
||||
// Scroll and extract company information
|
||||
|
||||
|
||||
|
||||
// Get all article elements
|
||||
const articles_l = page.locator('div[role="article"]');
|
||||
|
||||
|
||||
// while (article_count < 30) {
|
||||
// await page.evaluate((element) => {
|
||||
// element.scrollTop = element.scrollHeight;
|
||||
// }, await page.locator('div[role="feed"]').elementHandle());
|
||||
// await page.waitForLoadState('networkidle');
|
||||
// article_count = await page.locator('div[role="article"]').count();
|
||||
// }
|
||||
|
||||
await waitUser();
|
||||
|
||||
const articles = await articles_l.all();
|
||||
|
||||
let map = new Map();
|
||||
let map_e = new Map();
|
||||
// Extract information from each article
|
||||
for (const article of articles) {
|
||||
try {
|
||||
// Get company name
|
||||
const nameElement = article.locator('div.fontHeadlineSmall').first();
|
||||
const name_tt = nameElement ? await nameElement.textContent() : 'N/A';
|
||||
const name = name_tt?.trim();
|
||||
|
||||
if (!name) continue;
|
||||
// Get website URL
|
||||
const linkElement = article.locator('a[data-value="Spletno mesto"]');
|
||||
let url: string | null = 'N/A';
|
||||
if (await linkElement.count() > 0) {
|
||||
url = await linkElement.first().getAttribute('href');
|
||||
if(!url) continue;
|
||||
if(url[0] == '/'){
|
||||
await page2.goto(`https://www.google.com${url}`);
|
||||
await page2.waitForLoadState('networkidle');
|
||||
url = page2.url()
|
||||
}
|
||||
}
|
||||
if (url != 'N/A')
|
||||
map.set(name, url);
|
||||
else
|
||||
map_e.set(name, url);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error processing article:', error);
|
||||
}
|
||||
}
|
||||
|
||||
let smap = JSON.stringify(Array.from(map.entries()), null, 2);
|
||||
Bun.write("company.json", smap);
|
||||
|
||||
let smap_e = JSON.stringify(Array.from(map_e.entries()), null, 2);
|
||||
Bun.write("company_e.json", smap_e);
|
||||
|
||||
console.log(`Num of company ${map.size}`);
|
||||
console.log(`Num of company_e ${map_e.size}`);
|
||||
console.log('Finished scraping.');
|
||||
|
||||
|
||||
await context.close();
|
||||
await browser.close();
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
|
||||
|
||||
const file_map = Bun.file("company.json");
|
||||
const file_map_d = Bun.file("company_d.json");
|
||||
|
||||
let smap = "[]";
|
||||
if(await file_map.exists())
|
||||
smap = await file_map.text();
|
||||
|
||||
let smap_d = "[]";
|
||||
if(await file_map_d.exists())
|
||||
smap_d = await file_map_d.text();
|
||||
|
||||
|
||||
|
||||
let map = new Map<string, string>(JSON.parse(smap))
|
||||
let map_d = new Map<string, number>(JSON.parse(smap_d));
|
||||
|
||||
for (const [name, url] of map) {
|
||||
if(map_d.has(name)) continue;
|
||||
|
||||
const res = await Bun.$`./get_company.sh ${url}`.text();
|
||||
console.log(res);
|
||||
map_d.set(name, 1);
|
||||
break;
|
||||
}
|
||||
|
||||
let smap_e = JSON.stringify(Array.from(map_d.entries()), null, 2);
|
||||
Bun.write("company_d.json", smap_e);
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "job",
|
||||
"module": "index.ts",
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5"
|
||||
},
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"playwright": "^1.59.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
// Environment setup & latest features
|
||||
"lib": ["ESNext"],
|
||||
"target": "ESNext",
|
||||
"module": "Preserve",
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"allowJs": true,
|
||||
|
||||
// Bundler mode
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"noEmit": true,
|
||||
|
||||
// Best practices
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
|
||||
// Some stricter flags (disabled by default)
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"noPropertyAccessFromIndexSignature": false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user