CDP with Remote webdriver. 'WebDriver' object has no attribute 'execute_cdp_cmd' python

Question

I'm trying to run tests with CDP,

webdriver.execute_cdp_cmd('Network.enable', {}) 

with Remote webdriver (in Selenoid). But getting this error:

AttributeError: 'WebDriver' object has no attribute 'execute_cdp_cmd'. In local environment it works fine. I've tried to use Selenium 3.141.0 and 4.1.3.

I'm familiar with PyCDP documentation (https://py-cdp.readthedocs.io/en/latest/getting_started.html) but I didn't figured out how to properly use it.

Why it does not work with Remote webdriver? Do someone have an example of executing CDP commands using python in Selenium 4?

I use following capabilities, :

capabilities = { 'loggingPrefs': {'browser': 'ALL'}, 'goog:loggingPrefs': {'performance': 'ALL'}, "browserName": "chrome", "browserVersion": "99.0", "selenoid:options": { "enableVNC": True, "enableVideo": False } }

if request.config.getoption('--remote'): driver = webdriver.Remote(command_executor='selenoid.dev:4444/wd/hub', desired_capabilities=capabilities, options=options)

Answer 1

The official way to use CDP in Python's Selenium library is to use bidirectional functionality

It's an asyncio API. However, you can turn it into a sync API by using trio. I use a wrapper function to make it easier to use.

from selenium import webdriver
# You may need to change this if you need to use different version of CDP
import selenium.webdriver.common.devtools.v111 as devtools
import trio

def execute_cdp(driver: webdriver.Remote, cmd):
    async def execute_cdp_async():
        async with driver.bidi_connection() as session:
            cdp_session = session.session
            return await cdp_session.execute(cmd)
    # It will have error if we use asyncio.run
    # https://github.com/SeleniumHQ/selenium/issues/11244
    return trio.run(execute_cdp_async)

# Use it this way:
execute_cdp(driver, devtools.network.enable())
mhtml = execute_cdp(driver, devtools.page.capture_snapshot())

Answer 2

In 2024, it looks like CDP is working for Remote Drivers.

My code example with the solution of how I am using it for getting a specific 3rd party API call response:

import json
import logging
import sys

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.remote.webdriver import WebDriver

logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logger = logging.getLogger(__name__)


def prepare_browser() -> WebDriver:
    chrome_options = Options()
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--disable-dev-shm-usage")
    chrome_options.add_argument("--window-size=1920,1080")
    chrome_options.add_experimental_option(
        "prefs",
        {
            "intl.accept_languages": "en,en_US",
            "profile.managed_default_content_settings.images": 2,
        },
    )
    chrome_options.set_capability("goog:loggingPrefs", {"performance": "ALL"})
    chrome_options.set_capability("browserVersion", "latest")
    chrome_options.set_capability(
        "selenoid:options", {"enableVNC": True, "enableVideo": False}
    )

    return webdriver.Remote(
        command_executor="http://<selenoid-host>:4444/wd/hub",
        options=chrome_options,
    )


def get_browser_request_body(driver: WebDriver, request_id: str) -> str:
    browser_response = driver.execute(
        driver_command="executeCdpCommand",
        params={
            "cmd": "Network.getResponseBody",
            "params": {"requestId": request_id},
        },
    )
    return browser_response["value"]["body"].split("\n")[-1]


def get_browser_performance_logs(driver: WebDriver) -> list[dict]:
    browser_response = driver.execute(
        driver_command="getLog", params={"type": "performance"}
    )
    return browser_response["value"]


def intercept_json_by_url_part(driver: WebDriver, url_part: str) -> str | None:
    performance_logs = get_browser_performance_logs(driver=driver)

    for log in performance_logs:
        message = log["message"]

        if "Network.response" not in log["message"]:
            continue

        params = json.loads(message)["message"].get("params")
        response = params.get("response") if params else None

        if response and url_part in response["url"]:
            logger.info(f"Found required url part in url: {response['url']}")
            return get_browser_request_body(
                driver=driver, request_id=params["requestId"]
            )


def main() -> None:
    driver = prepare_browser()

    driver.get("https://demo.realworld.io")

    response = intercept_json_by_url_part(driver=driver, url_part="/api/tags")
    print(response)

    """
    Response:
    {"tags":["est","enim","ipsum","repellat","exercitationem","eos","quia","tenetur","facilis","consequatur"]}
    """


if __name__ == "__main__":
    main()

Important Note (2024-03-28):

My code example above will work starting from Selenium 4.16.0

For any reason if you want to work with Remote Connection via Chrome for Selenium 4.0.0 < selenium < 4.16.0 you should configure the Remote Connection class for Chrome:

from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.chromium.remote_connection import ChromiumRemoteConnection


class ChromeRemoteConnection(ChromiumRemoteConnection):
    browser_name = DesiredCapabilities.CHROME["browserName"]

    def __init__(
        self,
        remote_server_addr: str,
        keep_alive: bool = True,
        ignore_proxy: bool = False,
    ) -> None:
        super().__init__(
            remote_server_addr=remote_server_addr,
            vendor_prefix="goog",
            browser_name=ChromeRemoteConnection.browser_name,
            keep_alive=keep_alive,
            ignore_proxy=ignore_proxy,
        )


def prepare_browser() -> WebDriver:
    chrome_options = Options()
    chrome_options.add_argument("--no-sandbox")
    chrome_options.add_argument("--disable-dev-shm-usage")
    chrome_options.add_argument("--window-size=1920,1080")
    chrome_options.add_experimental_option(
        "prefs",
        {
            "intl.accept_languages": "en,en_US",
            "profile.managed_default_content_settings.images": 2,
        },
    )
    chrome_options.set_capability("goog:loggingPrefs", {"performance": "ALL"})
    chrome_options.set_capability("browserVersion", "latest")
    chrome_options.set_capability("browserName", "chrome")
    chrome_options.set_capability(
        "selenoid:options", {"enableVNC": True, "enableVideo": False}
    )

    command_executor = ChromeRemoteConnection(
        remote_server_addr="http://<selenoid-host>:4444/wd/hub"
    )
    return webdriver.Remote(
        command_executor=command_executor, options=chrome_options
    )

Answer 3

Looks like CDP is not supported for remote webdrivers.

Found this sweet workaround the issue:

import json

def send(driver, cmd, params={}):
  resource = "/session/%s/chromium/send_command_and_get_result" % driver.session_id
  url = driver.command_executor._url + resource
  body = json.dumps({'cmd': cmd, 'params': params})
  response = driver.command_executor._request('POST', url, body)
  return response.get('value')

send(webdriver, 'Network.enable', {})

source and relevant discussion: https://github.com/SeleniumHQ/selenium/issues/8672