Test PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT setting

Author: elacuesta

README.md

@@ -228,4 +228,5 @@ class ScrollSpider(scrapy.Spider):
         page = response.meta["playwright_page"]
         await page.screenshot(options={"path": "quotes.png", "fullPage": True})
         yield {"quote_count": len(response.css("div.quote"))}  # quotes from several pages
+        await page.close()
 ```

tests/mockserver.py

@@ -1,5 +1,6 @@
 import re
 import sys
+import time
 from http.server import HTTPServer, BaseHTTPRequestHandler
 from pathlib import Path
 from subprocess import Popen, PIPE
@@ -28,19 +29,31 @@ def urljoin(self, url):
         return urljoin("http://{}:{}".format(self.address, self.port), url)
 
 
-class _PostRequestHandler(BaseHTTPRequestHandler):
+class _RequestHandler(BaseHTTPRequestHandler):
     def do_POST(self):
+        """
+        Echo back the request body
+        """
         content_length = int(self.headers["Content-Length"])
         body = self.rfile.read(content_length)
         self.send_response(200)
         self.end_headers()
         self.wfile.write(b"Request body: ")
         self.wfile.write(body)
 
+    def do_GET(self):
+        """
+        Take a long time to reply
+        """
+        time.sleep(3)
+        self.send_response(200)
+        self.end_headers()
+        self.wfile.write(b"Hello world!")
+
 
-class PostMockServer:
+class MockServer:
     def __enter__(self):
-        self.httpd = HTTPServer(("127.0.0.1", 0), _PostRequestHandler)
+        self.httpd = HTTPServer(("127.0.0.1", 0), _RequestHandler)
         self.address, self.port = self.httpd.server_address
         self.thread = Thread(target=self.httpd.serve_forever)
         self.thread.start()

tests/test_playwright_requests.py

@@ -12,7 +12,7 @@
 from scrapy_playwright.handler import ScrapyPlaywrightDownloadHandler
 from scrapy_playwright.page import PageCoroutine as PageCoro
 
-from tests.mockserver import PostMockServer, StaticMockServer
+from tests.mockserver import MockServer, StaticMockServer
 
 
 def get_mimetype(file):
@@ -57,7 +57,7 @@ async def test_post_request(self):
         )
         await handler._launch_browser()
 
-        with PostMockServer() as server:
+        with MockServer() as server:
             req = FormRequest(
                 server.urljoin("/"), meta={"playwright": True}, formdata={"foo": "bar"}
             )
@@ -132,6 +132,25 @@ async def test_page_coroutine_infinite_scroll(self):
 
         await handler.browser.close()
 
+    @pytest.mark.asyncio
+    async def test_timeout(self):
+        handler = ScrapyPlaywrightDownloadHandler(
+            get_crawler(
+                settings_dict={
+                    "PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
+                    "PLAYWRIGHT_DEFAULT_NAVIGATION_TIMEOUT": 1000,
+                }
+            )
+        )
+        await handler._launch_browser()
+
+        with MockServer() as server:
+            req = Request(server.urljoin("/index.html"), meta={"playwright": True})
+            with pytest.raises(TimeoutError):
+                await handler._download_request(req, Spider("foo"))
+
+        await handler.browser.close()
+
     @pytest.mark.asyncio
     async def test_context_args(self):
         handler = ScrapyPlaywrightDownloadHandler(