fix: resolve issue with sub link scrapping

Author: FeckNeck

packages/scraper/src/index.ts

@@ -1,9 +1,9 @@
-import { writeFile } from 'fs/promises';
+import { writeFile, mkdir } from 'fs/promises';
 import { Worker } from 'worker_threads';
 import { weapons } from './weapons.js';
 import type { Skin } from '@mhgu-wtt/shared';
 
-const MAX_WORKERS = 12;
+const MAX_WORKERS = 4;
 let activeWorkers = 0;
 
 function startWorker(
@@ -58,6 +58,6 @@ async function processWeapons() {
 }
 
 await processWeapons();
-
+await mkdir('./data', { recursive: true });
 await writeFile('./data/weapons.json', JSON.stringify(weapons, null, 2));
 console.log('✅ File saved successfully!');

packages/scraper/src/worker.ts

@@ -11,30 +11,44 @@ await page.goto(`${KIRANICO_URL}/${weaponPrefix}`);
 
 const skins: Skin[] = [];
 
-const skinLinks = await page.locator('table').first().locator('a').all();
+const skinElements = await page.locator('td[rowspan]').all();
+
 const skinUrls = await Promise.all(
-  skinLinks.map((skinLink: Locator) => skinLink.getAttribute('href')),
+  skinElements.map(async (td: Locator) => {
+    try {
+      const firstLink = td.locator('a').first();
+      return await firstLink.getAttribute('href');
+    } catch (error) {
+      const innerText = await td.innerText();
+      console.log(
+        `Weapon ${weaponId} - Error getting skin URL for ${innerText}:`,
+        error,
+      );
+      return null;
+    }
+  }),
 );
 
-for (const skinUrl of skinUrls) {
-  if (skinUrl) {
-    const skinPage = await browser.newPage();
-    await skinPage.goto(skinUrl);
-
-    const skinName = await skinPage.locator('rt').first().innerText();
-    const skinId = await skinPage.evaluate(() => {
-      const div = document.querySelector('div#model-viewer.center-block');
-      if (div) {
-        const bgImage = window.getComputedStyle(div).backgroundImage;
-        return bgImage.replace(/\D/g, '');
-      }
-      return 'Not found';
-    });
-
-    console.log(`Weapon ${weaponId} - Skin ${skinId}: ${skinName}`);
-    skins.push({ id: skinId, name: skinName });
-    await skinPage.close();
-  }
+const filteredSkinUrls = skinUrls.filter((url) => url !== null);
+console.log(`Found ${filteredSkinUrls.length} skins for weapon ${weaponId}`);
+
+for (const skinUrl of filteredSkinUrls) {
+  const skinPage = await browser.newPage();
+  await skinPage.goto(skinUrl);
+
+  const skinName = await skinPage.locator('rt').first().innerText();
+  const skinId = await skinPage.evaluate(() => {
+    const div = document.querySelector('div#model-viewer.center-block');
+    if (div) {
+      const bgImage = window.getComputedStyle(div).backgroundImage;
+      return bgImage.replace(/\D/g, '');
+    }
+    return 'Not found';
+  });
+
+  console.log(`Weapon ${weaponId} - Skin ${skinId}: ${skinName}`);
+  skins.push({ id: skinId, name: skinName });
+  await skinPage.close();
 }
 
 await page.close();