Merge pull request #3268 from seleniumbase/cdp-mode-patch-10

CDP Mode - Patch 10

Author: mdmintz

examples/cdp_mode/ReadMe.md

@@ -323,19 +323,22 @@ sb.cdp.get_active_element_css()
 sb.cdp.click(selector)
 sb.cdp.click_active_element()
 sb.cdp.click_if_visible(selector)
+sb.cdp.click_visible_elements(selector)
 sb.cdp.mouse_click(selector)
 sb.cdp.nested_click(parent_selector, selector)
 sb.cdp.get_nested_element(parent_selector, selector)
-sb.cdp.flash(selector)
+sb.cdp.select_option_by_text(dropdown_selector, option)
+sb.cdp.flash(selector, duration=1, color="44CC88", pause=0)
+sb.cdp.highlight(selector)
 sb.cdp.focus(selector)
 sb.cdp.highlight_overlay(selector)
 sb.cdp.remove_element(selector)
 sb.cdp.remove_from_dom(selector)
 sb.cdp.remove_elements(selector)
-sb.cdp.scroll_into_view(selector)
 sb.cdp.send_keys(selector, text)
 sb.cdp.press_keys(selector, text)
 sb.cdp.type(selector, text)
+sb.cdp.set_value(selector, text)
 sb.cdp.evaluate(expression)
 sb.cdp.js_dumps(obj_name)
 sb.cdp.maximize()
@@ -373,6 +376,11 @@ sb.cdp.gui_press_keys(keys)
 sb.cdp.gui_write(text)
 sb.cdp.gui_click_x_y(x, y)
 sb.cdp.gui_click_element(selector)
+sb.cdp.gui_drag_drop_points(x1, y1, x2, y2)
+sb.cdp.gui_drag_and_drop(drag_selector, drop_selector)
+sb.cdp.gui_hover_x_y(x, y)
+sb.cdp.gui_hover_element(selector)
+sb.cdp.gui_hover_and_click(hover_selector, click_selector)
 sb.cdp.internalize_links()
 sb.cdp.is_checked(selector)
 sb.cdp.is_selected(selector)
@@ -382,12 +390,20 @@ sb.cdp.uncheck_if_checked(selector)
 sb.cdp.unselect_if_selected(selector)
 sb.cdp.is_element_present(selector)
 sb.cdp.is_element_visible(selector)
-sb.cdp.assert_element(selector)
 sb.cdp.assert_element_present(selector)
+sb.cdp.assert_element_absent(selector)
+sb.cdp.assert_element(selector)
+sb.cdp.assert_element_visible(selector)
+sb.cdp.assert_element_not_visible(selector)
+sb.cdp.assert_title(title)
 sb.cdp.assert_text(text, selector="html")
 sb.cdp.assert_exact_text(text, selector="html")
-sb.cdp.scroll_down(amount=25)
+sb.cdp.scroll_into_view(selector)
+sb.cdp.scroll_to_y(y)
+sb.cdp.scroll_to_top()
+sb.cdp.scroll_to_bottom()
 sb.cdp.scroll_up(amount=25)
+sb.cdp.scroll_down(amount=25)
 sb.cdp.save_screenshot(name, folder=None, selector=None)
 ```
 

examples/cdp_mode/raw_albertsons.py

@@ -28,7 +28,7 @@
         item.scroll_into_view()
         sb.sleep(0.025)
         if required_text in item.text:
-            item.flash()
+            item.flash(color="44CC88")
             sb.sleep(0.025)
             if item.text not in unique_item_text:
                 unique_item_text.append(item.text)

examples/cdp_mode/raw_browserscan.py

@@ -0,0 +1,10 @@
+from seleniumbase import SB
+
+with SB(uc=True, test=True, ad_block=True) as sb:
+    url = "https://www.browserscan.net/bot-detection"
+    sb.activate_cdp_mode(url)
+    sb.sleep(1)
+    sb.cdp.flash("Test Results", duration=4)
+    sb.sleep(1)
+    sb.cdp.assert_element('strong:contains("Normal")')
+    sb.cdp.flash('strong:contains("Normal")', duration=4, pause=4)

examples/cdp_mode/raw_cdp_with_sb.py

@@ -6,7 +6,7 @@
 with SB(uc=True, test=True, locale_code="en") as sb:
     url = "https://www.priceline.com/"
     sb.activate_cdp_mode(url)
-    sb.sleep(3)
+    sb.sleep(2.5)
     sb.internalize_links()  # Don't open links in a new tab
     sb.click("#link_header_nav_experiences")
     sb.sleep(2.5)

examples/cdp_mode/raw_footlocker.py

@@ -3,7 +3,7 @@
 with SB(uc=True, test=True, locale_code="en") as sb:
     url = "https://www.footlocker.com/"
     sb.activate_cdp_mode(url)
-    sb.sleep(3)
+    sb.sleep(2.5)
     sb.cdp.click_if_visible('button[id*="Agree"]')
     sb.sleep(1.5)
     sb.cdp.mouse_click('input[aria-label="Search"]')

examples/cdp_mode/raw_nike.py

@@ -3,7 +3,7 @@
 with SB(uc=True, test=True, locale_code="en") as sb:
     url = "https://www.nike.com/"
     sb.activate_cdp_mode(url)
-    sb.sleep(3)
+    sb.sleep(2.5)
     sb.cdp.gui_click_element('div[data-testid="user-tools-container"]')
     sb.sleep(1.5)
     search = "Nike Air Force 1"

examples/cdp_mode/raw_planetmc.py

@@ -3,6 +3,6 @@
 with SB(uc=True, test=True, locale_code="en") as sb:
     url = "www.planetminecraft.com/account/sign_in/"
     sb.activate_cdp_mode(url)
-    sb.sleep(1)
-    sb.cdp.gui_click_element("#turnstile-widget")
+    sb.sleep(2)
+    sb.cdp.gui_click_element("#turnstile-widget div")
     sb.sleep(2)

examples/cdp_mode/raw_pokemon.py

@@ -3,22 +3,24 @@
 with SB(uc=True, test=True, locale_code="en") as sb:
     url = "https://www.pokemon.com/us"
     sb.activate_cdp_mode(url)
-    sb.sleep(3)
+    sb.sleep(2.5)
     sb.cdp.click_if_visible("button#onetrust-reject-all-handler")
-    sb.sleep(1)
+    sb.sleep(1.2)
     sb.cdp.click('a[href="https://www.pokemon.com/us/pokedex/"]')
-    sb.sleep(1)
+    sb.sleep(1.2)
     sb.cdp.click('b:contains("Show Advanced Search")')
-    sb.sleep(1)
+    sb.sleep(1.2)
     sb.cdp.click('span[data-type="type"][data-value="electric"]')
-    sb.sleep(1)
+    sb.sleep(0.5)
+    sb.scroll_into_view("a#advSearch")
+    sb.sleep(0.5)
     sb.cdp.click("a#advSearch")
-    sb.sleep(1)
+    sb.sleep(1.2)
     sb.cdp.click('img[src*="img/pokedex/detail/025.png"]')
     sb.cdp.assert_text("Pikachu", 'div[class*="title"]')
     sb.cdp.assert_element('img[alt="Pikachu"]')
     sb.cdp.scroll_into_view("div.pokemon-ability-info")
-    sb.sleep(1)
+    sb.sleep(1.2)
     sb.cdp.flash('div[class*="title"]')
     sb.cdp.flash('img[alt="Pikachu"]')
     sb.cdp.flash("div.pokemon-ability-info")

examples/cdp_mode/raw_priceline.py

@@ -4,7 +4,7 @@
     window_handle = sb.driver.current_window_handle
     url = "https://www.priceline.com"
     sb.activate_cdp_mode(url)
-    sb.sleep(3)
+    sb.sleep(2.5)
     sb.cdp.click('input[name="endLocation"]')
     sb.sleep(1)
     location = "Portland, OR, USA"
@@ -16,14 +16,17 @@
     sb.cdp.click(selection)
     sb.sleep(1.5)
     sb.cdp.click('button[aria-label="Dismiss calendar"]')
-    sb.sleep(5)
+    sb.sleep(4.5)
     sb.connect()
     if len(sb.driver.window_handles) > 1:
         sb.switch_to_window(window_handle)
         sb.driver.close()
         sb.sleep(0.2)
         sb.switch_to_newest_window()
         sb.sleep(0.6)
+    for y in range(1, 9):
+        sb.scroll_to_y(y * 400)
+        sb.sleep(1.25)
     hotel_names = sb.find_elements('a[data-autobot-element-id*="HOTEL_NAME"]')
     hotel_prices = sb.find_elements('span[font-size="4,,,5"]')
     print("Priceline Hotels in %s:" % location)
@@ -34,4 +37,5 @@
     for i, hotel in enumerate(hotel_names):
         if hotel_prices[i] and hotel_prices[i].text:
             count += 1
-            print("* %s: %s => %s" % (count, hotel.text, hotel_prices[i].text))
+            hotel_price = "$" + hotel_prices[i].text
+            print("* %s: %s => %s" % (count, hotel.text, hotel_price))

examples/cdp_mode/raw_xhr_sb.py

@@ -0,0 +1,92 @@
+"""CDP.network.ResponseReceived with CDP.network.ResourceType.XHR."""
+import ast
+import asyncio
+import colorama
+import mycdp
+import sys
+import time
+from seleniumbase.undetected import cdp_driver
+
+xhr_requests = []
+last_xhr_request = None
+c1 = colorama.Fore.BLUE + colorama.Back.LIGHTYELLOW_EX
+c2 = colorama.Fore.BLUE + colorama.Back.LIGHTGREEN_EX
+cr = colorama.Style.RESET_ALL
+if "linux" in sys.platform:
+    c1 = c2 = cr = ""
+
+
+def listenXHR(page):
+    async def handler(evt):
+        # Get AJAX requests
+        if evt.type_ is mycdp.network.ResourceType.XHR:
+            xhr_requests.append([evt.response.url, evt.request_id])
+            global last_xhr_request
+            last_xhr_request = time.time()
+    page.add_handler(mycdp.network.ResponseReceived, handler)
+
+
+async def receiveXHR(page, requests):
+    responses = []
+    retries = 0
+    max_retries = 5
+    # Wait at least 2 seconds after last XHR request for more
+    while True:
+        if last_xhr_request is None or retries > max_retries:
+            break
+        if time.time() - last_xhr_request <= 2:
+            retries = retries + 1
+            time.sleep(2)
+            continue
+        else:
+            break
+    await page
+    # Loop through gathered requests and get response body
+    for request in requests:
+        try:
+            res = await page.send(mycdp.network.get_response_body(request[1]))
+            if res is None:
+                continue
+            responses.append({
+                "url": request[0],
+                "body": res[0],
+                "is_base64": res[1],
+            })
+        except Exception as e:
+            print("Error getting response:", e)
+    return responses
+
+
+async def crawl():
+    driver = await cdp_driver.cdp_util.start_async()
+    tab = await driver.get("about:blank")
+    listenXHR(tab)
+
+    # Change url to something that makes ajax requests
+    tab = await driver.get("https://resttesttest.com/")
+    time.sleep(1)
+    # Click AJAX button on https://resttesttest.com/
+    element = await tab.select("button#submitajax")
+    await element.click_async()
+    time.sleep(2)
+
+    xhr_responses = await receiveXHR(tab, xhr_requests)
+    for response in xhr_responses:
+        print(c1 + "*** ==> XHR Request URL <== ***" + cr)
+        print(f'{response["url"]}')
+        is_base64 = response["is_base64"]
+        b64_data = "Base64 encoded data"
+        try:
+            headers = ast.literal_eval(response["body"])["headers"]
+            print(c2 + "*** ==> XHR Response Headers <== ***" + cr)
+            print(headers if not is_base64 else b64_data)
+        except Exception:
+            response_body = response["body"]
+            print(c2 + "*** ==> XHR Response Body <== ***" + cr)
+            print(response_body if not is_base64 else b64_data)
+
+
+if __name__ == "__main__":
+    print("================= Starting =================")
+    loop = asyncio.new_event_loop()
+    loop.run_until_complete(crawl())

requirements.txt

@@ -1,16 +1,16 @@
 pip>=24.2
 packaging>=24.2
 setuptools~=70.2;python_version<"3.10"
-setuptools>=73.0.1;python_version>="3.10"
+setuptools>=75.5.0;python_version>="3.10"
 wheel>=0.45.0
 attrs>=24.2.0
 certifi>=2024.8.30
 exceptiongroup>=1.2.2
 websockets~=13.1;python_version<"3.9"
-websockets>=14.0;python_version>="3.9"
+websockets>=14.1;python_version>="3.9"
 filelock>=3.16.1
 fasteners>=0.19
-mycdp>=1.0.1
+mycdp>=1.1.0
 pynose>=1.5.3
 platformdirs>=4.3.6
 typing-extensions>=4.12.2
@@ -64,7 +64,7 @@ rich==13.9.4
 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.)
 
 coverage>=7.6.1;python_version<"3.9"
-coverage>=7.6.4;python_version>="3.9"
+coverage>=7.6.5;python_version>="3.9"
 pytest-cov>=5.0.0;python_version<"3.9"
 pytest-cov>=6.0.0;python_version>="3.9"
 flake8==5.0.4;python_version<"3.9"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.32.9"
+__version__ = "4.32.10"