t

scrape/index.ts

@@ -0,0 +1,94 @@
+import playwright from 'playwright';
+
+function sleep(ms: number) {
+  return Bun.sleep(ms);
+}
+
+async function waitUser() {
+  for await (const line of console) {
+    console.log(line);
+    // wait for user enter to continue
+    break;
+  }
+}
+
+
+const browser = await playwright.chromium.launch({ headless: false });
+const context = await browser.newContext();
+const page = await context.newPage();
+const page2 = await context.newPage();
+
+await context.route('**.jpg', route => route.abort());
+await page.goto('https://www.google.com/maps/search/Software+company/@46.0463511,14.5023435,14z');
+
+
+await page.click('span:has-text("Sprejmi")');
+await page.waitForLoadState('networkidle');
+
+// Scroll and extract company information
+
+
+
+// Get all article elements
+const articles_l = page.locator('div[role="article"]');
+
+
+// while (article_count < 30) {
+//   await page.evaluate((element) => {
+//     element.scrollTop = element.scrollHeight;
+//   }, await page.locator('div[role="feed"]').elementHandle());
+//   await page.waitForLoadState('networkidle');
+//   article_count = await page.locator('div[role="article"]').count();
+// }
+
+await waitUser();
+
+const articles = await articles_l.all();
+
+let map = new Map();
+let map_e = new Map();
+// Extract information from each article
+for (const article of articles) {
+  try {
+    // Get company name
+    const nameElement = article.locator('div.fontHeadlineSmall').first();
+    const name_tt = nameElement ? await nameElement.textContent() : 'N/A';
+    const name = name_tt?.trim();
+
+    if (!name) continue;
+    // Get website URL
+    const linkElement = article.locator('a[data-value="Spletno mesto"]');
+    let url: string | null = 'N/A';
+    if (await linkElement.count() > 0) {
+      url = await linkElement.first().getAttribute('href');
+      if(!url) continue;
+      if(url[0] == '/'){
+        await page2.goto(`https://www.google.com${url}`);
+        await page2.waitForLoadState('networkidle');
+        url = page2.url()
+      }
+    }
+    if (url != 'N/A')
+      map.set(name, url);
+    else
+      map_e.set(name, url);
+
+  } catch (error) {
+    console.error('Error processing article:', error);
+  }
+}
+
+let smap = JSON.stringify(Array.from(map.entries()), null, 2);
+Bun.write("company.json", smap);
+
+let smap_e = JSON.stringify(Array.from(map_e.entries()), null, 2);
+Bun.write("company_e.json", smap_e);
+
+console.log(`Num of company ${map.size}`);
+console.log(`Num of company_e ${map_e.size}`);
+console.log('Finished scraping.');
+
+
+await context.close();
+await browser.close();
+