Merge pull request #3693 from seleniumbase/improve-cdp-multithreading

Improve CDP Multithreading

Author: mdmintz

examples/cdp_mode/ReadMe.md

@@ -286,6 +286,7 @@ with SB(uc=True, test=True, ad_block=True) as sb:
     sb.activate_cdp_mode(url)
     sb.sleep(2.5)
     sb.cdp.click_if_visible('[data-automation-id*="close-mark"]')
+    sb.sleep(0.3)
     sb.cdp.mouse_click('input[aria-label="Search"]')
     sb.sleep(1.2)
     search = "Settlers of Catan Board Game"
@@ -300,7 +301,7 @@ with SB(uc=True, test=True, ad_block=True) as sb:
     for item in items:
         if required_text in item.text:
             description = item.querySelector(
-                '[data-automation-id="product-price"] + span'
+                '[data-automation-id="product-title"]'
             )
             if description and description.text not in unique_item_text:
                 unique_item_text.append(description.text)

examples/cdp_mode/raw_pixelscan.py

@@ -2,12 +2,20 @@
 
 with SB(uc=True, incognito=True, test=True) as sb:
     sb.activate_cdp_mode("https://pixelscan.net/")
-    sb.sleep(3)
-    sb.remove_elements(".bg-bannerBg")  # Remove the top banner
-    sb.remove_elements("pxlscn-ad1")  # Remove the ad banner
+    sb.sleep(2)
+    sb.click('button[class*="startButton"]')
+    sb.sleep(6)
+    sb.remove_elements(".bg-bannerBg")  # Remove top banner
+    sb.remove_elements("pxlscn-ad1")  # Remove an ad banner
+    sb.remove_elements("pxlscn-ad2")  # Remove an ad banner
     sb.remove_elements("jdiv")  # Remove chat widgets
+    sb.sleep(14)
     not_masking_text = "You are not masking your fingerprint"
-    sb.assert_text(not_masking_text, "pxlscn-fingerprint-masking")
+    sb.assert_text(
+        not_masking_text,
+        "pxlscn-fingerprint-masking",
+        timeout=20,
+    )
     no_automation_detected = "No automation framework detected"
     sb.assert_text(no_automation_detected, "pxlscn-bot-detection")
     consistent_selector = 'div.bg-consistentBg [alt="Good"]'

examples/cdp_mode/raw_theaters.py

@@ -5,6 +5,7 @@
     url = "https://architectureofcities.com/roman-theaters"
     sb.activate_cdp_mode(url)
     sb.cdp.click_if_visible("#cn-close-notice")
+    sb.cdp.click_if_visible('span:contains("Continue")')
     sb.sleep(1)
     print("*** " + sb.cdp.get_text("h1") + " ***")
     for item in sb.cdp.find_elements("h3"):

examples/cdp_mode/raw_walmart.py

@@ -5,6 +5,7 @@
     sb.activate_cdp_mode(url)
     sb.sleep(2.5)
     sb.cdp.click_if_visible('[data-automation-id*="close-mark"]')
+    sb.sleep(0.3)
     sb.cdp.mouse_click('input[aria-label="Search"]')
     sb.sleep(1.2)
     search = "Settlers of Catan Board Game"
@@ -19,7 +20,7 @@
     for item in items:
         if required_text in item.text:
             description = item.querySelector(
-                '[data-automation-id="product-price"] + span'
+                '[data-automation-id="product-title"]'
             )
             if description and description.text not in unique_item_text:
                 unique_item_text.append(description.text)

examples/presenter/uc_presentation_4.py

@@ -521,6 +521,7 @@ def test_presentation_4(self):
             sb.activate_cdp_mode(url)
             sb.sleep(2.5)
             sb.cdp.click_if_visible('[data-automation-id*="close-mark"]')
+            sb.sleep(0.3)
             sb.cdp.mouse_click('input[aria-label="Search"]')
             sb.sleep(1.2)
             search = "Settlers of Catan Board Game"
@@ -535,7 +536,7 @@ def test_presentation_4(self):
             for item in items:
                 if required_text in item.text:
                     description = item.querySelector(
-                        '[data-automation-id="product-price"] + span'
+                        '[data-automation-id="product-title"]'
                     )
                     if (
                         description

examples/raw_cdp_logging.py

@@ -6,7 +6,7 @@
     url = "seleniumbase.io/apps/turnstile"
     driver.uc_open_with_reconnect(url, 2)
     driver.uc_gui_handle_captcha()
-    driver.sleep(3)
+    driver.sleep(2)
     pprint(driver.get_log("performance"))
 finally:
     driver.quit()

examples/raw_pixelscan.py

@@ -1,12 +1,18 @@
 from seleniumbase import SB
 
 with SB(uc=True, incognito=True, test=True) as sb:
-    sb.driver.uc_open_with_reconnect("https://pixelscan.net/", 7)
-    sb.remove_elements(".bg-bannerBg")  # Remove the top banner
-    sb.remove_elements("pxlscn-ad1")  # Remove the ad banner
+    sb.driver.uc_open_with_reconnect("https://pixelscan.net/", 2)
+    sb.uc_click('button[class*="startButton"]', reconnect_time=20)
+    sb.remove_elements(".bg-bannerBg")  # Remove top banner
+    sb.remove_elements("pxlscn-ad1")  # Remove an ad banner
+    sb.remove_elements("pxlscn-ad2")  # Remove an ad banner
     sb.remove_elements("jdiv")  # Remove chat widgets
     no_automation_detected = "No automation framework detected"
-    sb.assert_text(no_automation_detected, "pxlscn-bot-detection")
+    sb.assert_text(
+        no_automation_detected,
+        "pxlscn-bot-detection",
+        timeout=20,
+    )
     not_masking_text = "You are not masking your fingerprint"
     sb.assert_text(not_masking_text, "pxlscn-fingerprint-masking")
     consistent_selector = 'div.bg-consistentBg [alt="Good"]'

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.37.5"
+__version__ = "4.37.6"

seleniumbase/undetected/__init__.py

@@ -402,7 +402,8 @@ def get(self, url):
 
     def add_cdp_listener(self, event_name, callback):
         if (
-            self.reactor
+            hasattr(self, "reactor")
+            and self.reactor
             and self.reactor is not None
             and isinstance(self.reactor, Reactor)
         ):
@@ -411,7 +412,11 @@ def add_cdp_listener(self, event_name, callback):
         return False
 
     def clear_cdp_listeners(self):
-        if self.reactor and isinstance(self.reactor, Reactor):
+        if (
+            hasattr(self, "reactor")
+            and self.reactor
+            and isinstance(self.reactor, Reactor)
+        ):
             self.reactor.handlers.clear()
 
     def window_new(self, url=None):
@@ -581,7 +586,11 @@ def quit(self):
                 finally:
                     with suppress(Exception):
                         self.service._terminate_process()
-        if self.reactor and hasattr(self.reactor, "event"):
+        if (
+            hasattr(self, "reactor")
+            and self.reactor
+            and hasattr(self.reactor, "event")
+        ):
             logger.debug("Shutting down Reactor")
             with suppress(Exception):
                 self.reactor.event.set()

seleniumbase/undetected/cdp_driver/connection.py

@@ -292,16 +292,16 @@ async def aclose(self):
         Closes the websocket connection. Shouldn't be called manually by users.
         """
         if self.websocket and self.websocket.state is not State.CLOSED:
-            if self.listener and self.listener.running:
-                self.listener.cancel()
-                self.enabled_domains.clear()
             try:
                 await self.websocket.close()
             except Exception:
                 logger.debug(
                     "\n❌ Error closing websocket connection to %s",
                     self.websocket_url
                 )
+            if self.listener and self.listener.running:
+                self.listener.cancel()
+                self.enabled_domains.clear()
             logger.debug(
                 "\n❌ Closed websocket connection to %s", self.websocket_url
             )
@@ -549,8 +549,7 @@ async def listener_loop(self):
             except asyncio.TimeoutError:
                 self.idle.set()
                 # Pause for a moment.
-                # await asyncio.sleep(self.time_before_considered_idle / 10)
-                await asyncio.sleep(0.015)
+                await asyncio.sleep(self.time_before_considered_idle / 10)
                 continue
             except (Exception,) as e:
                 logger.debug(