95 lines
2.5 KiB
TypeScript
95 lines
2.5 KiB
TypeScript
import playwright from 'playwright';
|
|
|
|
function sleep(ms: number) {
|
|
return Bun.sleep(ms);
|
|
}
|
|
|
|
async function waitUser() {
|
|
for await (const line of console) {
|
|
console.log(line);
|
|
// wait for user enter to continue
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
const browser = await playwright.chromium.launch({ headless: false });
|
|
const context = await browser.newContext();
|
|
const page = await context.newPage();
|
|
const page2 = await context.newPage();
|
|
|
|
await context.route('**.jpg', route => route.abort());
|
|
await page.goto('https://www.google.com/maps/search/Software+company/@46.0463511,14.5023435,14z');
|
|
|
|
|
|
await page.click('span:has-text("Sprejmi")');
|
|
await page.waitForLoadState('networkidle');
|
|
|
|
// Scroll and extract company information
|
|
|
|
|
|
|
|
// Get all article elements
|
|
const articles_l = page.locator('div[role="article"]');
|
|
|
|
|
|
// while (article_count < 30) {
|
|
// await page.evaluate((element) => {
|
|
// element.scrollTop = element.scrollHeight;
|
|
// }, await page.locator('div[role="feed"]').elementHandle());
|
|
// await page.waitForLoadState('networkidle');
|
|
// article_count = await page.locator('div[role="article"]').count();
|
|
// }
|
|
|
|
await waitUser();
|
|
|
|
const articles = await articles_l.all();
|
|
|
|
let map = new Map();
|
|
let map_e = new Map();
|
|
// Extract information from each article
|
|
for (const article of articles) {
|
|
try {
|
|
// Get company name
|
|
const nameElement = article.locator('div.fontHeadlineSmall').first();
|
|
const name_tt = nameElement ? await nameElement.textContent() : 'N/A';
|
|
const name = name_tt?.trim();
|
|
|
|
if (!name) continue;
|
|
// Get website URL
|
|
const linkElement = article.locator('a[data-value="Spletno mesto"]');
|
|
let url: string | null = 'N/A';
|
|
if (await linkElement.count() > 0) {
|
|
url = await linkElement.first().getAttribute('href');
|
|
if(!url) continue;
|
|
if(url[0] == '/'){
|
|
await page2.goto(`https://www.google.com${url}`);
|
|
await page2.waitForLoadState('networkidle');
|
|
url = page2.url()
|
|
}
|
|
}
|
|
if (url != 'N/A')
|
|
map.set(name, url);
|
|
else
|
|
map_e.set(name, url);
|
|
|
|
} catch (error) {
|
|
console.error('Error processing article:', error);
|
|
}
|
|
}
|
|
|
|
let smap = JSON.stringify(Array.from(map.entries()), null, 2);
|
|
Bun.write("company.json", smap);
|
|
|
|
let smap_e = JSON.stringify(Array.from(map_e.entries()), null, 2);
|
|
Bun.write("company_e.json", smap_e);
|
|
|
|
console.log(`Num of company ${map.size}`);
|
|
console.log(`Num of company_e ${map_e.size}`);
|
|
console.log('Finished scraping.');
|
|
|
|
|
|
await context.close();
|
|
await browser.close();
|
|
|