Skip to content

Commit e401902

Browse files
mdmintzmdmintz
authored
Merge pull request #3224 from seleniumbase/cdp-mode-patch-3
CDP Mode - Patch 3
2 parents 32f1288 + 1aaee33 commit e401902

26 files changed

+287
-125
lines changed

‎examples/cdp_mode/ReadMe.md

+16-15
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
## [<img src="https://seleniumbase.github.io/img/logo6.png" title="SeleniumBase" width="32">](https://github.com/seleniumbase/SeleniumBase/) CDP Mode 🐙
44

5-
🐙 <b translate="no">SeleniumBase</b> <b translate="no">CDP Mode</b> (Chrome Devtools Protocol Mode) is a special mode inside of <b><a href="https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/uc_mode.md" translate="no"><span translate="no">SeleniumBase UC Mode</span></a></b> that lets bots appear human while controlling the browser with the <b translate="no">CDP-Driver</b>. Although regular <span translate="no">UC Mode</span> can't perform <span translate="no">WebDriver</span> actions while the <code>driver</code> is disconnected from the browser, the <span translate="no">CDP-Driver</span> can still perform actions (while maintaining its cover).
5+
🐙 <b translate="no">SeleniumBase</b> <b translate="no">CDP Mode</b> (Chrome Devtools Protocol Mode) is a special mode inside of <b><a href="https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/uc_mode.md" translate="no"><span translate="no">SeleniumBase UC Mode</span></a></b> that lets bots appear human while controlling the browser with the <b translate="no">CDP-Driver</b>. Although regular <span translate="no">UC Mode</span> can't perform <span translate="no">WebDriver</span> actions while the <code>driver</code> is disconnected from the browser, the <span translate="no">CDP-Driver</span> can still perform actions while maintaining its cover. (For Python 3.11 or newer!)
66

77
👤 <b translate="no">UC Mode</b> avoids bot-detection by first disconnecting WebDriver from the browser at strategic times, calling special <code>PyAutoGUI</code> methods to bypass CAPTCHAs (as needed), and finally reconnecting the <code>driver</code> afterwards so that WebDriver actions can be performed again. Although this approach works for bypassing simple CAPTCHAs, more flexibility is needed for bypassing bot-detection on websites with advanced protection. (That's where <b translate="no">CDP Mode</b> comes in.)
88

@@ -74,15 +74,15 @@ from seleniumbase import SB
7474
with SB(uc=True, test=True, locale_code="en") as sb:
7575
url = "https://www.pokemon.com/us"
7676
sb.activate_cdp_mode(url)
77-
sb.sleep(1.5)
77+
sb.sleep(3)
7878
sb.cdp.click_if_visible("button#onetrust-reject-all-handler")
79-
sb.sleep(0.5)
79+
sb.sleep(1)
8080
sb.cdp.click('a[href="https://www.pokemon.com/us/pokedex/"]')
8181
sb.sleep(1)
8282
sb.cdp.click('b:contains("Show Advanced Search")')
8383
sb.sleep(1)
8484
sb.cdp.click('span[data-type="type"][data-value="electric"]')
85-
sb.sleep(0.5)
85+
sb.sleep(1)
8686
sb.cdp.click("a#advSearch")
8787
sb.sleep(1)
8888
sb.cdp.click('img[src*="img/pokedex/detail/025.png"]')
@@ -130,13 +130,13 @@ from seleniumbase import SB
130130
with SB(uc=True, test=True, locale_code="en") as sb:
131131
url = "https://www.hyatt.com/"
132132
sb.activate_cdp_mode(url)
133-
sb.sleep(1.5)
133+
sb.sleep(2)
134134
sb.cdp.click_if_visible('button[aria-label="Close"]')
135-
sb.sleep(0.5)
135+
sb.sleep(1)
136136
sb.cdp.click('span:contains("Explore")')
137137
sb.sleep(1)
138138
sb.cdp.click('a:contains("Hotels & Resorts")')
139-
sb.sleep(2.5)
139+
sb.sleep(3)
140140
location = "Anaheim, CA, USA"
141141
sb.cdp.press_keys("input#searchbox", location)
142142
sb.sleep(1)
@@ -145,7 +145,7 @@ with SB(uc=True, test=True, locale_code="en") as sb:
145145
sb.cdp.click('div.hotel-card-footer button')
146146
sb.sleep(1)
147147
sb.cdp.click('button[data-locator="find-hotels"]')
148-
sb.sleep(4)
148+
sb.sleep(5)
149149
hotel_names = sb.cdp.select_all(
150150
'div[data-booking-status="BOOKABLE"] [class*="HotelCard_header"]'
151151
)
@@ -177,20 +177,20 @@ from seleniumbase import SB
177177
with SB(uc=True, test=True, locale_code="en") as sb:
178178
url = "https://www.bestwestern.com/en_US.html"
179179
sb.activate_cdp_mode(url)
180-
sb.sleep(1.5)
180+
sb.sleep(2.5)
181181
sb.cdp.click_if_visible("div.onetrust-close-btn-handler")
182-
sb.sleep(0.5)
182+
sb.sleep(1)
183183
sb.cdp.click("input#destination-input")
184-
sb.sleep(1.5)
184+
sb.sleep(2)
185185
location = "Palm Springs, CA, USA"
186186
sb.cdp.press_keys("input#destination-input", location)
187-
sb.sleep(0.6)
187+
sb.sleep(1)
188188
sb.cdp.click("ul#google-suggestions li")
189-
sb.sleep(0.6)
189+
sb.sleep(1)
190190
sb.cdp.click("button#btn-modify-stay-update")
191-
sb.sleep(1.5)
192-
sb.cdp.click("label#available-label")
193191
sb.sleep(4)
192+
sb.cdp.click("label#available-label")
193+
sb.sleep(2.5)
194194
print("Best Western Hotels in %s:" % location)
195195
summary_details = sb.cdp.get_text("#summary-details-column")
196196
dates = summary_details.split("ROOM")[0].split("DATES")[-1].strip()
@@ -288,6 +288,7 @@ sb.cdp.get_document()
288288
sb.cdp.get_flattened_document()
289289
sb.cdp.get_element_attributes(selector)
290290
sb.cdp.get_element_html(selector)
291+
sb.cdp.set_locale(locale)
291292
sb.cdp.set_attributes(selector, attribute, value)
292293
sb.cdp.internalize_links()
293294
sb.cdp.is_element_present(selector)

‎examples/cdp_mode/raw_async.py

+18-6
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
import asyncio
22
import time
3+
from contextlib import suppress
34
from seleniumbase.core import sb_cdp
45
from seleniumbase.undetected import cdp_driver
56

67

78
async def main():
89
driver = await cdp_driver.cdp_util.start_async()
9-
page = await driver.get("https://www.priceline.com/")
10+
page = await driver.get("about:blank")
11+
await page.set_locale("en")
12+
await page.get("https://www.priceline.com/")
1013
time.sleep(3)
1114
print(await page.evaluate("document.title"))
1215
element = await page.select('[data-testid*="endLocation"]')
@@ -22,24 +25,33 @@ async def main():
2225

2326
# Call everything without using async / await
2427
driver = cdp_driver.cdp_util.start_sync()
25-
page = loop.run_until_complete(driver.get("https://www.pokemon.com/us"))
28+
page = loop.run_until_complete(driver.get("about:blank"))
29+
loop.run_until_complete(page.set_locale("en"))
30+
loop.run_until_complete(page.get("https://www.pokemon.com/us"))
2631
time.sleep(3)
2732
print(loop.run_until_complete(page.evaluate("document.title")))
33+
with suppress(Exception):
34+
selector = "button#onetrust-reject-all-handler"
35+
element = loop.run_until_complete(page.select(selector, timeout=1))
36+
loop.run_until_complete(element.click_async())
37+
time.sleep(1)
2838
element = loop.run_until_complete(page.select("span.icon_pokeball"))
2939
loop.run_until_complete(element.click_async())
30-
time.sleep(1)
40+
time.sleep(1.5)
3141
print(loop.run_until_complete(page.evaluate("document.title")))
3242
time.sleep(1)
3343

3444
# Call CDP methods via the simplified CDP API
35-
page = loop.run_until_complete(driver.get("https://www.priceline.com/"))
45+
page = loop.run_until_complete(driver.get("about:blank"))
3646
sb = sb_cdp.CDPMethods(loop, page, driver)
47+
sb.set_locale("en")
48+
sb.open("https://www.priceline.com/")
3749
sb.sleep(3)
3850
sb.internalize_links() # Don't open links in a new tab
3951
sb.click("#link_header_nav_experiences")
40-
sb.sleep(2)
52+
sb.sleep(2.5)
4153
sb.remove_elements("msm-cookie-banner")
42-
sb.sleep(1)
54+
sb.sleep(1.5)
4355
sb.press_keys('input[data-test-id*="search"]', "Amsterdam")
4456
sb.sleep(2)
4557
sb.click('span[data-test-id*="autocomplete"]')

‎examples/cdp_mode/raw_bestwestern.py

+7-7
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,20 @@
33
with SB(uc=True, test=True, locale_code="en") as sb:
44
url = "https://www.bestwestern.com/en_US.html"
55
sb.activate_cdp_mode(url)
6-
sb.sleep(1.5)
6+
sb.sleep(2.5)
77
sb.cdp.click_if_visible("div.onetrust-close-btn-handler")
8-
sb.sleep(0.5)
8+
sb.sleep(1)
99
sb.cdp.click("input#destination-input")
10-
sb.sleep(1.5)
10+
sb.sleep(2)
1111
location = "Palm Springs, CA, USA"
1212
sb.cdp.press_keys("input#destination-input", location)
13-
sb.sleep(0.6)
13+
sb.sleep(1)
1414
sb.cdp.click("ul#google-suggestions li")
15-
sb.sleep(0.6)
15+
sb.sleep(1)
1616
sb.cdp.click("button#btn-modify-stay-update")
17-
sb.sleep(1.5)
18-
sb.cdp.click("label#available-label")
1917
sb.sleep(4)
18+
sb.cdp.click("label#available-label")
19+
sb.sleep(2.5)
2020
print("Best Western Hotels in %s:" % location)
2121
summary_details = sb.cdp.get_text("#summary-details-column")
2222
dates = summary_details.split("ROOM")[0].split("DATES")[-1].strip()

‎examples/cdp_mode/raw_cdp.py

+7-6
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,22 @@
88

99
@decorators.print_runtime("CDP Priceline Example")
1010
def main():
11-
url = "https://www.priceline.com/"
11+
url0 = "about:blank" # Set Locale code from here first
12+
url1 = "https://www.priceline.com/" # (The "real" URL)
1213
loop = asyncio.new_event_loop()
1314
driver = cdp_driver.cdp_util.start_sync()
14-
page = loop.run_until_complete(driver.get(url))
15+
page = loop.run_until_complete(driver.get(url0))
1516
sb = sb_cdp.CDPMethods(loop, page, driver)
17+
sb.set_locale("en") # This test expects English locale
18+
sb.open(url1)
1619
sb.sleep(3)
1720
sb.internalize_links() # Don't open links in a new tab
1821
sb.click("#link_header_nav_experiences")
19-
sb.sleep(2)
22+
sb.sleep(2.5)
2023
sb.remove_elements("msm-cookie-banner")
21-
sb.sleep(1)
24+
sb.sleep(1.5)
2225
location = "Amsterdam"
2326
sb.press_keys('input[data-test-id*="search"]', location)
24-
sb.sleep(1)
25-
sb.click('input[data-test-id*="search"]')
2627
sb.sleep(2)
2728
sb.click('span[data-test-id*="autocomplete"]')
2829
sb.sleep(5)

‎examples/cdp_mode/raw_cdp_with_sb.py

+3-5
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,17 @@
33
from seleniumbase import SB
44

55

6-
with SB(uc=True, test=True) as sb:
6+
with SB(uc=True, test=True, locale_code="en") as sb:
77
url = "https://www.priceline.com/"
88
sb.activate_cdp_mode(url)
99
sb.sleep(3)
1010
sb.internalize_links() # Don't open links in a new tab
1111
sb.click("#link_header_nav_experiences")
12-
sb.sleep(2)
12+
sb.sleep(2.5)
1313
sb.remove_elements("msm-cookie-banner")
14-
sb.sleep(1)
14+
sb.sleep(1.5)
1515
location = "Amsterdam"
1616
sb.press_keys('input[data-test-id*="search"]', location)
17-
sb.sleep(1)
18-
sb.click('input[data-test-id*="search"]')
1917
sb.sleep(2)
2018
sb.click('span[data-test-id*="autocomplete"]')
2119
sb.sleep(5)

‎examples/cdp_mode/raw_footlocker.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
sb.cdp.press_keys('input[aria-label="Search"]', search)
1313
sb.sleep(2)
1414
sb.cdp.mouse_click('ul[id*="typeahead"] li div')
15-
sb.sleep(2)
15+
sb.sleep(3)
1616
elements = sb.cdp.select_all("a.ProductCard-link")
1717
if elements:
1818
print('**** Found results for "%s": ****' % search)

‎examples/cdp_mode/raw_hyatt.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@
33
with SB(uc=True, test=True, locale_code="en") as sb:
44
url = "https://www.hyatt.com/"
55
sb.activate_cdp_mode(url)
6-
sb.sleep(1.5)
6+
sb.sleep(2)
77
sb.cdp.click_if_visible('button[aria-label="Close"]')
8-
sb.sleep(0.5)
8+
sb.sleep(1)
99
sb.cdp.click('span:contains("Explore")')
1010
sb.sleep(1)
1111
sb.cdp.click('a:contains("Hotels & Resorts")')
12-
sb.sleep(2.5)
12+
sb.sleep(3)
1313
location = "Anaheim, CA, USA"
1414
sb.cdp.press_keys("input#searchbox", location)
1515
sb.sleep(1)
@@ -18,7 +18,7 @@
1818
sb.cdp.click('div.hotel-card-footer button')
1919
sb.sleep(1)
2020
sb.cdp.click('button[data-locator="find-hotels"]')
21-
sb.sleep(4)
21+
sb.sleep(5)
2222
hotel_names = sb.cdp.select_all(
2323
'div[data-booking-status="BOOKABLE"] [class*="HotelCard_header"]'
2424
)

‎examples/cdp_mode/raw_pokemon.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,15 @@
33
with SB(uc=True, test=True, locale_code="en") as sb:
44
url = "https://www.pokemon.com/us"
55
sb.activate_cdp_mode(url)
6-
sb.sleep(1.5)
6+
sb.sleep(3)
77
sb.cdp.click_if_visible("button#onetrust-reject-all-handler")
8-
sb.sleep(0.5)
8+
sb.sleep(1)
99
sb.cdp.click('a[href="https://www.pokemon.com/us/pokedex/"]')
1010
sb.sleep(1)
1111
sb.cdp.click('b:contains("Show Advanced Search")')
1212
sb.sleep(1)
1313
sb.cdp.click('span[data-type="type"][data-value="electric"]')
14-
sb.sleep(0.5)
14+
sb.sleep(1)
1515
sb.cdp.click("a#advSearch")
1616
sb.sleep(1)
1717
sb.cdp.click('img[src*="img/pokedex/detail/025.png"]')

‎examples/cdp_mode/raw_priceline.py

+7-7
Original file line numberDiff line numberDiff line change
@@ -4,26 +4,26 @@
44
window_handle = sb.driver.current_window_handle
55
url = "https://www.priceline.com"
66
sb.activate_cdp_mode(url)
7-
sb.sleep(1)
7+
sb.sleep(3)
88
sb.cdp.click('input[name="endLocation"]')
99
sb.sleep(1)
1010
location = "Portland, OR, USA"
1111
selection = "Oregon, United States" # (Dropdown option)
1212
sb.cdp.press_keys('input[name="endLocation"]', location)
1313
sb.sleep(1)
1414
sb.click_if_visible('input[name="endLocation"]')
15-
sb.sleep(1)
16-
sb.cdp.click("Oregon, United States")
17-
sb.sleep(1)
15+
sb.sleep(0.5)
16+
sb.cdp.click(selection)
17+
sb.sleep(1.5)
1818
sb.cdp.click('button[aria-label="Dismiss calendar"]')
19-
sb.sleep(3)
19+
sb.sleep(5)
2020
sb.connect()
2121
if len(sb.driver.window_handles) > 1:
2222
sb.switch_to_window(window_handle)
2323
sb.driver.close()
24-
sb.sleep(0.1)
24+
sb.sleep(0.2)
2525
sb.switch_to_newest_window()
26-
sb.sleep(1)
26+
sb.sleep(0.6)
2727
hotel_names = sb.find_elements('a[data-autobot-element-id*="HOTEL_NAME"]')
2828
hotel_prices = sb.find_elements('span[font-size="4,,,5"]')
2929
print("Priceline Hotels in %s:" % location)

‎examples/cdp_mode/raw_req_async.py

+3-4
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,11 @@ class RequestPausedTest():
99
async def request_paused_handler(self, event, tab):
1010
r = event.request
1111
is_image = ".png" in r.url or ".jpg" in r.url or ".gif" in r.url
12-
is_blocked = True if is_image else False
13-
if not is_blocked:
12+
if not is_image: # Let the data through
1413
tab.feed_cdp(
1514
mycdp.fetch.continue_request(request_id=event.request_id)
1615
)
17-
else:
16+
else: # Block the data (images)
1817
TIMED_OUT = mycdp.network.ErrorReason.TIMED_OUT
1918
s = f"BLOCKING | {r.method} | {r.url}"
2019
print(f" >>> ------------\n{s}")
@@ -23,7 +22,7 @@ async def request_paused_handler(self, event, tab):
2322
)
2423

2524
async def start_test(self):
26-
driver = await cdp_driver.cdp_util.start_async(incognito=True)
25+
driver = await cdp_driver.cdp_util.start_async()
2726
tab = await driver.get("about:blank")
2827
tab.add_handler(mycdp.fetch.RequestPaused, self.request_paused_handler)
2928
url = "https://gettyimages.com/photos/firefly-2003-nathan"

‎examples/cdp_mode/raw_req_sb.py

+3-4
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,16 @@
66
async def request_paused_handler(event, tab):
77
r = event.request
88
is_image = ".png" in r.url or ".jpg" in r.url or ".gif" in r.url
9-
is_blocked = True if is_image else False
10-
if not is_blocked:
9+
if not is_image: # Let the data through
1110
tab.feed_cdp(mycdp.fetch.continue_request(request_id=event.request_id))
12-
else:
11+
else: # Block the data (images)
1312
TIMED_OUT = mycdp.network.ErrorReason.TIMED_OUT
1413
s = f"BLOCKING | {r.method} | {r.url}"
1514
print(f" >>> ------------\n{s}")
1615
tab.feed_cdp(mycdp.fetch.fail_request(event.request_id, TIMED_OUT))
1716

1817

19-
with SB(uc=True, test=True, locale_code="en", incognito=True) as sb:
18+
with SB(uc=True, test=True, locale_code="en") as sb:
2019
sb.activate_cdp_mode("about:blank")
2120
sb.cdp.add_handler(mycdp.fetch.RequestPaused, request_paused_handler)
2221
url = "https://gettyimages.com/photos/firefly-2003-nathan"

‎examples/raw_gui_click.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
with SB(uc=True, test=True, rtf=True, agent=agent) as sb:
1010
url = "https://gitlab.com/users/sign_in"
11-
sb.activate_cdp_mode(url)
11+
sb.uc_open_with_reconnect(url)
1212
sb.uc_gui_click_captcha() # Only if needed
1313
sb.assert_element('label[for="user_login"]')
1414
sb.assert_element('input[data-testid*="username"]')

‎examples/raw_pyautogui.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
with SB(uc=True, test=True, rtf=True, agent=agent) as sb:
1010
url = "https://gitlab.com/users/sign_in"
11-
sb.uc_open_with_reconnect(url, 4)
11+
sb.uc_open_with_reconnect(url)
1212
sb.uc_gui_handle_captcha() # Only if needed
1313
sb.assert_element('label[for="user_login"]')
1414
sb.assert_element('input[data-testid*="username"]')

‎examples/raw_recaptcha.py

-2
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,13 @@
22

33
with SB(uc=True, test=True) as sb:
44
url = "https://seleniumbase.io/apps/recaptcha"
5-
sb.activate_cdp_mode(url)
65
sb.uc_gui_handle_captcha() # Try with TAB + SPACEBAR
76
sb.assert_element("img#captcha-success", timeout=3)
87
sb.set_messenger_theme(location="top_left")
98
sb.post_message("SeleniumBase wasn't detected", duration=3)
109

1110
with SB(uc=True, test=True) as sb:
1211
url = "https://seleniumbase.io/apps/recaptcha"
13-
sb.activate_cdp_mode(url)
1412
sb.uc_gui_click_captcha('iframe[src*="/recaptcha/"]')
1513
sb.assert_element("img#captcha-success", timeout=3)
1614
sb.set_messenger_theme(location="top_left")

0 commit comments

Comments
 (0)