Iterate

Author: serefyarar

Dockerfile

@@ -1,4 +1,4 @@
-FROM apify/actor-node-puppeteer-chrome:latest
+FROM apify/actor-node-playwright:latest
 COPY package*.json ./
 COPY yarn.lock ./
 RUN yarn install

index.js

@@ -1,6 +1,6 @@
 import express from "express";
 import { Actor } from "apify";
-import { PuppeteerCrawler, RequestList, sleep } from "crawlee";
+import { PlaywrightCrawler, RequestList, sleep } from "crawlee";
 
 const app = express();
 const port = 3000;
@@ -18,32 +18,25 @@ const initializeCrawler = async () => {
   });
   const requestQueue = await Actor.openRequestQueue();
 
-  const crawler = new PuppeteerCrawler({
+  const crawler = new PlaywrightCrawler({
     requestList,
     requestQueue,
-    useSessionPool: false,
+    useSessionPool: true,
     persistCookiesPerSession: false,
-    headless: true,
+    headless: false,
     keepAlive: true,
     minConcurrency: 5,
-    maxConcurrency: 15,
-    launchContext: {
-      launchOptions: {
-        defaultViewport: {
-          width: 1512,
-          height: 982,
-        },
-      },
-      userAgent:
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
-    },
+    maxConcurrency: 30,
     requestHandler: async ({ request, page }) => {
-      await Promise.race([
-        page.waitForNetworkIdle({ idleTime: 500, concurrency: 3 }),
-        new Promise((_, reject) =>
-          setTimeout(() => reject(new Error("Timeout")), 10000),
-        ),
-      ]);
+      await page.route('**/*', (route) => {
+        if (route.request().resourceType() === 'image') {
+          route.abort();
+        } else {
+          route.continue();
+        }
+      });
+
+      //sleep(60000);
 
       await page.evaluate(() => {
         return window.scrollBy(0, window.innerHeight);
@@ -60,7 +53,7 @@ const initializeCrawler = async () => {
       console.log(`Title: ${await page.title()}`);
       console.log(`Content: ${content}`);
       contentMap.set(request.uniqueKey, content); // Store content with uniqueKey
-      await requestQueue.markRequestHandled(request);
+      //await requestQueue.markRequestHandled(request);
     },
   });
 

package.json

@@ -11,6 +11,6 @@
     "apify": "^3.2.0",
     "crawlee": "^3.10.1",
     "express": "^4.19.2",
-    "puppeteer": "^22.9.0"
+    "playwright": "^1.44.1",
   }
 }

yarn.lock

@@ -119,7 +119,7 @@
     tiny-typed-emitter "^2.1.0"
     tslib "^2.4.0"
 
-"@crawlee/[email protected]":
+"@crawlee/[email protected]", "@crawlee/browser@^3.10.1":
   version "3.10.1"
   resolved "https://registry.yarnpkg.com/@crawlee/browser/-/browser-3.10.1.tgz#a7d5416a95b2accfbd751bbcbc8ff4808a09f901"
   integrity sha512-FgQbWQcIe787w8HhKYGjf4j3E64OmOJYkoKJa/KdWfSJ03PozAc3Bu8Kw1dSO8J0OOK8JpoBO5SnqazXcbjuzg==
@@ -252,7 +252,7 @@
     proper-lockfile "^4.1.2"
     tslib "^2.4.0"
 
-"@crawlee/[email protected]":
+"@crawlee/[email protected]", "@crawlee/playwright@^3.10.1":
   version "3.10.1"
   resolved "https://registry.yarnpkg.com/@crawlee/playwright/-/playwright-3.10.1.tgz#874f8d375023a06ab5361ee457ac3415a7f143c8"
   integrity sha512-NjYlqSVJO31zwoKrjIgce23BYl1rs9+nLPAM9Ppf5fFtEgMPONRCrHSg8WzICXrqT3yX0vU2lxKLTF2qwBdsGQ==
@@ -1402,6 +1402,11 @@ fs-extra@^11.0.0, fs-extra@^11.2.0:
     jsonfile "^6.0.1"
     universalify "^2.0.0"
 
+[email protected]:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+  integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
 function-bind@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c"
@@ -2277,6 +2282,20 @@ pkg-dir@^4.2.0:
   dependencies:
     find-up "^4.0.0"
 
+[email protected]:
+  version "1.44.1"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.44.1.tgz#53ec975503b763af6fc1a7aa995f34bc09ff447c"
+  integrity sha512-wh0JWtYTrhv1+OSsLPgFzGzt67Y7BE/ZS3jEqgGBlp2ppp1ZDj8c+9IARNW4dwf1poq5MgHreEM2KV/GuR4cFA==
+
+playwright@^1.44.1:
+  version "1.44.1"
+  resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.44.1.tgz#5634369d777111c1eea9180430b7a184028e7892"
+  integrity sha512-qr/0UJ5CFAtloI3avF95Y0L1xQo6r3LQArLIg/z/PoGJ6xa+EwzrwO5lpNr/09STxdHuUoP2mvuELJS+hLdtgg==
+  dependencies:
+    playwright-core "1.44.1"
+  optionalDependencies:
+    fsevents "2.3.2"
+
 [email protected]:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"