Update and refactor CDP Mode

Author: mdmintz

seleniumbase/core/sb_cdp.py

@@ -795,15 +795,13 @@ def click_if_visible(self, selector, timeout=0):
         if self.is_element_visible(selector):
             with suppress(Exception):
                 self.click(selector, timeout=1)
+        elif timeout == 0:
+            return
         else:
             with suppress(Exception):
-                element = self.find_element(selector, timeout=timeout)
-                self.sleep(0.1)
-                element.scroll_into_view()
-                self.sleep(0.1)
-                element.click()
-                self.__slow_mode_pause_if_set()
-                self.loop.run_until_complete(self.page.wait())
+                self.find_element(selector, timeout=timeout)
+                if self.is_element_visible(selector):
+                    self.click(selector, timeout=1)
 
     def click_visible_elements(self, selector, limit=0):
         """Finds all matching page elements and clicks visible ones in order.
@@ -2068,14 +2066,35 @@ def __cdp_click_incapsula_hcaptcha(self):
         time.sleep(0.05)
         x_offset = 30
         y_offset = 36
+        was_clicked = False
         gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK)
         with gui_lock:  # Prevent issues with multiple processes
             self.bring_active_window_to_front()
-            time.sleep(0.05)
+            time.sleep(0.056)
+            if "--debug" in sys.argv:
+                displayed_selector = "`%s`" % selector
+                if '"' not in selector:
+                    displayed_selector = '"%s"' % selector
+                elif "'" not in selector:
+                    displayed_selector = "'%s'" % selector
+                print(
+                    " <DEBUG> click_with_offset(%s, %s, %s)"
+                    % (displayed_selector, x_offset, y_offset)
+                )
             with suppress(Exception):
                 element.click_with_offset(x_offset, y_offset)
-                time.sleep(0.2)
-                return True
+                was_clicked = True
+                time.sleep(0.056)
+        if was_clicked:
+            # Wait a moment for the click to succeed
+            time.sleep(0.25)
+            self.__slow_mode_pause_if_set()
+            self.loop.run_until_complete(self.page.wait())
+            if "--debug" in sys.argv:
+                print(" <DEBUG> hCaptcha was clicked!")
+            return True
+        if "--debug" in sys.argv:
+            print(" <DEBUG> hCaptcha was NOT clicked!")
         return False
 
     def solve_captcha(self):

seleniumbase/fixtures/base_case.py

@@ -92,6 +92,7 @@ def test_anything(self):
 
 logging.getLogger("requests").setLevel(logging.ERROR)
 logging.getLogger("urllib3").setLevel(logging.ERROR)
+logging.getLogger("websocket").setLevel(logging.CRITICAL)
 urllib3.disable_warnings()
 LOGGER.setLevel(logging.WARNING)
 is_linux = shared_utils.is_linux()

seleniumbase/undetected/cdp_driver/browser.py

@@ -182,7 +182,6 @@ def stopped(self):
         if self._process and self._process.returncode is None:
             return False
         return True
-        # return (self._process and self._process.returncode) or False
 
     async def wait(self, time: Union[float, int] = 1) -> Browser:
         """Wait for <time> seconds. Important to use,
@@ -436,6 +435,7 @@ async def get(
                     "*.adthrive.com*",
                     "*.pubmatic.com*",
                     "*.id5-sync.com*",
+                    "*.moatads.com*",
                     "*.dotomi.com*",
                     "*.adsrvr.org*",
                     "*.atmtd.com*",
@@ -572,7 +572,7 @@ async def start(self=None) -> Browser:
                     If you are sure about the browser executable,
                     set it using `browser_executable_path='{}` parameter."""
                     ).format(
-                        "/path/to/browser/executable"
+                        "/path/to/your/browser/executable"
                         if is_posix
                         else "c:/path/to/your/browser.exe"
                     )
@@ -592,8 +592,6 @@ async def start(self=None) -> Browser:
         if not connect_existing:
             self._process: asyncio.subprocess.Process = (
                 await asyncio.create_subprocess_exec(
-                    # self.config.browser_executable_path,
-                    # *cmdparams,
                     exe,
                     *params,
                     stdin=asyncio.subprocess.PIPE,
@@ -602,19 +600,23 @@ async def start(self=None) -> Browser:
                     close_fds=is_posix,
                 )
             )
+            await asyncio.sleep(0.05)
             self._process_pid = self._process.pid
+        await asyncio.sleep(0.05)
         self._http = HTTPApi((self.config.host, self.config.port))
+        await asyncio.sleep(0.05)
         get_registered_instances().add(self)
-        await asyncio.sleep(0.25)
-        for _ in range(5):
+        await asyncio.sleep(0.15)
+        for attempt in range(5):
             try:
                 self.info = ContraDict(
                     await self._http.get("version"), silent=True
                 )
             except (Exception,):
-                if _ == 4:
+                if attempt == 4:
                     logger.debug("Could not start", exc_info=True)
-                await self.sleep(0.5)
+                else:
+                    await self.sleep(0.5)
             else:
                 break
         if not self.info:

seleniumbase/undetected/cdp_driver/connection.py

@@ -7,6 +7,7 @@
 import logging
 import sys
 import types
+import warnings
 from typing import (
     Optional,
     Generator,
@@ -445,11 +446,17 @@ async def send(
             if not _is_update:
                 await self._register_handlers()
             await self.websocket.send(tx.message)
-            try:
-                return await tx
-            except ProtocolException as e:
-                e.message += f"\ncommand:{tx.method}\nparams:{tx.params}"
-                raise e
+            with warnings.catch_warnings():
+                warnings.filterwarnings(
+                    "ignore",
+                    message="coroutine .* was never awaited",
+                    category=RuntimeWarning,
+                )
+                try:
+                    return await tx
+                except ProtocolException as e:
+                    e.message += f"\ncommand:{tx.method}\nparams:{tx.params}"
+                    raise e
         except Exception:
             await self.aclose()