Edits
- I changed the title and forgot to change the question. Woops.
- In further testing, I noticed a message in the console that said to disable LittleProxy pass
--use-littleproxy false. When I used it, my Selenium test succeeds, and my HTTP Archive has a mostly hydrated Request and Response, but the Response content is missing, despite flags added to capture everything. - Added an HTTP Archive generated by BrowserMob Proxy, and another from Chrome Dev Tools to demonstrate the differences
Question
I'm attempting to export an HTTP Archive (HAR) file representing the automated Selenium test for diagnostic purposes, but either I get an error when using the Proxy, or the WebDriver instance ignores the proxy entirely. How can I fix this?
Problem
I get the following error from BrowserMob Proxy (stand-alone running in LittleProxy mode)
[ERROR 2024-01-05T10:28:10,203 org.littleshoot.proxy.impl.ProxyToServerConnection] (LittleProxy-159-ProxyToServerWorker-3) (DISCONNECTED) [id: 0x55b60925, L:0.0.0.0/0.0.0.0:51669 ! R:esp.aptrinsic.com/35.184.35.160:443]: Caught an exception on ProxyToServerConnection io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
...
Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
...
I have done nothing to the configuration of BrowserMob Proxy. I downloaded it from the latest release, and used the provided browsermob-proxy.bat script to kick it off. As for the code that routes Selenium to the proxy, see below.
Code
Note: I work in a corporate environment that prevents me from sharing verbatim the code I used. As such, this is a stripped down version of the supporting code.
const ProxyClient = require('browsermob-proxy-client');
const { Browser, Builder, By, Capabilities, Key } = require('selenium-webdriver');
const { Options: ChromeOptions } = require('selenium-webdriver/chrome');
const { Options: EdgeOptions } = require('selenium-webdriver/edge');
const { Options: FirefoxOptions } = require('selenium-webdriver/firefox');
const { Options: SafariOptions } = require('selenium-webdriver/safari');
const { manual, system } = require('selenium-webdriver/proxy');
const valid = Object.entries(Browser).reduce((map, [k, v]) => map.set(k, v).set(v, v), new Map());
function sleep(milliseconds) {
const start = new Date();
return new Promise(resolve => {
const interval = setInterval(() => console.log('sleep elapsed =>', Date.now() - start), 500);
setTimeout(() => {
clearInterval(interval);
resolve();
}, milliseconds);
});
}
async function createProxy(skip = false) {
if(skip) return {};
// Creates the helper for HTTP calls to BrowserMob Proxy
const proxy = ProxyClient.createClient({ host: 'localhost', port: 8080, protocol: 'http' });
// Starts the client instance, and returns the port number
const info = await proxy.start({ trustAllServers: true });
// Starts the HTTP Archive trace
await proxy.createHar({
captureBinaryContent: true,
captureContent: true,
captureCookies: true,
captureHeaders: true
});
console.log('Proxy Info =>', info);
const url = `localhost:${info.port}`;
const proxyConfig = manual({ http: url, https: url });
return { proxy, proxyConfig };
}
async function disposeProxy(proxy) {
if(!proxy) return;
const har = await proxy.getHar();
console.dir(har, { depth: 6 });
await proxy.closeProxies();
}
function getChromeBuilder(proxy) {
const capabilities = Capabilities.chrome();
const options = new ChromeOptions();
if(proxy) capabilities.setAcceptInsecureCerts(true).setProxy(proxy);
return new Builder().forBrowser(Browser.CHROME).setChromeOptions(options).withCapabilities(capabilities);
}
function getEdgeBuilder(proxy) {
const capabilities = Capabilities.edge();
const options = new EdgeOptions();
if(proxy) capabilities.setAcceptInsecureCerts(true).setProxy(proxy);
return new Builder().forBrowser(Browser.EDGE).setEdgeOptions(options).withCapabilities(capabilities);
}
function getFirefoxBuilder(proxy) {
const capabilities = Capabilities.firefox();
const options = new FirefoxOptions();
if(proxy) capabilities.setAcceptInsecureCerts(true).setProxy(proxy);
return new Builder().forBrowser(Browser.FIREFOX).setFirefoxOptions(options).withCapabilities(capabilities);
}
function getSafariBuilder(proxy) {
const capabilities = Capabilities.safari();
const options = new SafariOptions();
if(proxy) capabilities.setAcceptInsecureCerts(true).setProxy(proxy);
return new Builder().forBrowser(Browser.SAFARI).setSafariOptions(options).withCapabilities(capabilities);
}
function getBuilder(browser, proxy = system()) {
switch(browser) {
case Browser.EDGE:
return getEdgeBuilder(proxy);
case Browser.FIREFOX:
return getFirefoxBuilder(proxy);
case Browser.SAFARI:
return getSafariBuilder(proxy);
default:
return getChromeBuilder(proxy);
}
}
async function test(driver) {
await driver.get('https://www.google.com');
const searchBar = await driver.findElement(By.css('textarea[name=q]'));
await searchBar.sendKeys('do a barrel roll', Key.RETURN);
await sleep(5000);
await driver.quit();
}
async function main(browser, skipProxy = false) {
console.log('Enter main');
const { proxy, proxyConfig } = await createProxy(skipProxy);
const driver = await getBuilder(browser, proxyConfig).build();
await test(driver).catch(err => console.log('Test failed to execute.', err));
await disposeProxy(proxy).catch(err => console.log('Failed to dispose proxy.', err));
}
const [ , , browser = Browser.CHROME, skipProxyArg = '' ] = process.argv;
const validBrowser = valid.get(browser) ?? valid.get(browser.toLowerCase()) ?? valid.get(browser.toUpperCase()) ?? Browser.CHROME;
const skipProxy = /(1|true|yes)/i.test(skipProxyArg);
main(validBrowser, skipProxy).then(() => console.log('Done')).catch(err => console.warn('Unhandled exception.', err));
Some things of note are happening in this isolated code sample:
- Chromium-based browsers (so Chrome and Edge) attempt to use the proxy, but I get
WebDriverError: unknown error: net::ERR_TUNNEL_CONNECTION_FAILEDor sometimesWebDriverError: unknown error: net::ERR_PROXY_CONNECTION_FAILED - The HTTP Archive generated has an entry per request, but all responses are empty due to a failure to connect
- Firefox ignores the proxy configuration entirely
Remarks
I've been working on this for the better part of three weeks. The original work I did was back in Jan 2021, and it worked back then with very little effort beyond standing up BrowserMob Proxy, and that was mostly because I didn't read the documentation closely, and it required I download the JAR and run locally.
So, what am I doing wrong? Any help would be greatly appreciated.
Edit #2 Context
Here is an example entry from the HTTP Archive now that I switched to the legacy proxy provider.
{
"log": {
"version": "1.2",
"creator": {
"name": "BrowserMob Proxy",
"version": "2.1.4-legacy",
"comment": ""
},
"pages": [
{
"id": "Page 0",
"startedDateTime": "2024-01-05T15:23:14.226-05:00",
"title": "Page 0",
"pageTimings": {
"comment": ""
},
"comment": ""
}
"entries": [
{
"pageref": "Page 0",
"startedDateTime": "2024-01-05T15:23:16.183-05:00",
"request": {
"method": "GET",
"url": "https://www.google.com/?safe=active&ssui=on",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"queryString": [
{
"name": "ssui",
"value": "on"
},
{
"name": "safe",
"value": "active"
}
],
"headersSize": 700,
"bodySize": 0,
"comment": ""
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [],
"content": {
"size": 56290,
"mimeType": "text/html; charset=UTF-8",
"comment": ""
},
"redirectURL": "",
"headersSize": 2196,
"bodySize": 56290,
"comment": ""
},
"cache": {},
"timings": {
"comment": "",
"blocked": 0,
"dns": 28,
"connect": 29,
"send": 0,
"ssl": 16,
"receive": 117,
"wait": 188
},
"serverIPAddress": "64.233.185.104",
"comment": "",
"time": 363
}
],
"comment": ""
}
}
And here is the same request/response using Chrome Dev Tools. Note: I've modified some of the data to remove potentially identifying information, such as cookies identified as Secure. Additionally, anything that was base64-encoded, I sliced into two pieces, truncated the middle, and put an ellipsis between them.
{
"log": {
"pages": [
],
"entries": [
{
"_initiator": {
"type": "other"
},
"_priority": "VeryHigh",
"_resourceType": "document",
"cache": {},
"connection": "794164",
"pageref": "page_1",
"request": {
"method": "GET",
"url": "https://google.com/?safe=active&ssui=on",
"httpVersion": "HTTP/1.1",
"headers": [
{
"name": "Accept",
"value": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
},
{
"name": "Accept-Encoding",
"value": "gzip, deflate, br"
},
{
"name": "Accept-Language",
"value": "en-US,en;q=0.9"
},
{
"name": "Connection",
"value": "keep-alive"
},
{
"name": "Host",
"value": "www.google.com"
},
{
"name": "Sec-Fetch-Dest",
"value": "document"
},
{
"name": "Sec-Fetch-Mode",
"value": "navigate"
},
{
"name": "Sec-Fetch-Site",
"value": "none"
},
{
"name": "Sec-Fetch-User",
"value": "?1"
},
{
"name": "Upgrade-Insecure-Requests",
"value": "1"
},
{
"name": "User-Agent",
"value": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
},
{
"name": "sec-ch-ua",
"value": "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\""
},
{
"name": "sec-ch-ua-arch",
"value": "\"x86\""
},
{
"name": "sec-ch-ua-bitness",
"value": "\"64\""
},
{
"name": "sec-ch-ua-full-version",
"value": "\"120.0.6099.130\""
},
{
"name": "sec-ch-ua-full-version-list",
"value": "\"Not_A Brand\";v=\"8.0.0.0\", \"Chromium\";v=\"120.0.6099.130\", \"Google Chrome\";v=\"120.0.6099.130\""
},
{
"name": "sec-ch-ua-mobile",
"value": "?0"
},
{
"name": "sec-ch-ua-model",
"value": "\"\""
},
{
"name": "sec-ch-ua-platform",
"value": "\"Windows\""
},
{
"name": "sec-ch-ua-platform-version",
"value": "\"10.0.0\""
},
{
"name": "sec-ch-ua-wow64",
"value": "?0"
}
],
"queryString": [
{
"name": "safe",
"value": "active"
},
{
"name": "ssui",
"value": "on"
}
],
"cookies": [
{
"name": "1P_JAR",
"value": "2024-01-05-16",
"path": "/",
"domain": ".google.com",
"expires": "2024-02-04T16:18:34.592Z",
"httpOnly": false,
"secure": true,
"sameSite": "None"
},
{
"name": "NID",
"value": "511=SyoR8m7-p65I5vBw...7hFjR54yfx_1mZR5",
"path": "/",
"domain": ".google.com",
"expires": "2024-07-06T20:32:24.694Z",
"httpOnly": true,
"secure": true,
"sameSite": "None"
},
{
"name": "SIDCC",
"value": "ABTWhQFYVGxbJnkXhg...Z1goPb2KD6p4WQIts6A",
"path": "/",
"domain": ".google.com",
"expires": "2025-01-04T20:42:02.524Z",
"httpOnly": false,
"secure": false
}
],
"headersSize": 2773,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"headers": [
{
"name": "Accept-CH",
"value": "Sec-CH-UA-Platform"
},
{
"name": "Accept-CH",
"value": "Sec-CH-UA-Platform-Version"
},
{
"name": "Accept-CH",
"value": "Sec-CH-UA-Full-Version"
},
{
"name": "Accept-CH",
"value": "Sec-CH-UA-Arch"
},
{
"name": "Accept-CH",
"value": "Sec-CH-UA-Model"
},
{
"name": "Accept-CH",
"value": "Sec-CH-UA-Bitness"
},
{
"name": "Accept-CH",
"value": "Sec-CH-UA-Full-Version-List"
},
{
"name": "Accept-CH",
"value": "Sec-CH-UA-WoW64"
},
{
"name": "Cache-Control",
"value": "private, max-age=0"
},
{
"name": "Content-Encoding",
"value": "br"
},
{
"name": "Content-Length",
"value": "58451"
},
{
"name": "Content-Security-Policy-Report-Only",
"value": "object-src 'none';base-uri 'self';script-src 'nonce-A3PFCgdrZLDZrKQMR2_FpQ' 'strict-dynamic' 'report-sample' 'unsafe-eval' 'unsafe-inline' https: http:;report-uri https://csp.withgoogle.com/csp/gws/other-hp"
},
{
"name": "Content-Type",
"value": "text/html; charset=UTF-8"
},
{
"name": "Cross-Origin-Opener-Policy",
"value": "same-origin-allow-popups; report-to=\"gws\""
},
{
"name": "Date",
"value": "Fri, 05 Jan 2024 20:42:08 GMT"
},
{
"name": "Expires",
"value": "-1"
},
{
"name": "P3P",
"value": "CP=\"This is not a P3P policy! See g.co/p3phelp for more info.\""
},
{
"name": "Permissions-Policy",
"value": "unload=()"
},
{
"name": "Report-To",
"value": "{\"group\":\"gws\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/gws/other\"}]}"
},
{
"name": "Server",
"value": "gws"
},
{
"name": "Set-Cookie",
"value": "1P_JAR=2024-01-05-20; expires=Sun, 04-Feb-2024 20:42:08 GMT; path=/; domain=.google.com; Secure; SameSite=none"
},
{
"name": "Set-Cookie",
"value": "OTZ=; expires=Mon, 01-Jan-1990 00:00:00 GMT; path=/; domain=www.google.com"
},
{
"name": "Set-Cookie",
"value": "OTZ=; expires=Mon, 01-Jan-1990 00:00:00 GMT; path=/; domain=.www.google.com"
},
{
"name": "Set-Cookie",
"value": "OTZ=; expires=Mon, 01-Jan-1990 00:00:00 GMT; path=/; domain=google.com"
},
{
"name": "Set-Cookie",
"value": "OTZ=; expires=Mon, 01-Jan-1990 00:00:00 GMT; path=/; domain=.google.com"
},
{
"name": "Set-Cookie",
"value": "AEC=Ackid1QwcPoX9Lo...Qv2Qn6_kbwSMZabs1Dmjgjhcbcs; expires=Tue, 07-May-2024 17:19:43 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=lax"
},
{
"name": "Set-Cookie",
"value": "NID=511=SyoR8m7-p65I5vBw...7hFjR54yfx_1mZR5; expires=Sat, 06-Jul-2024 20:32:24 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=none"
},
{
"name": "Set-Cookie",
"value": "SIDCC=ABTWhQFYVGxbJnkXhg...Z1goPb2KD6p4WQIts6A; expires=Sat, 04-Jan-2025 20:42:08 GMT; path=/; domain=.google.com; priority=high"
},
{
"name": "Strict-Transport-Security",
"value": "max-age=31536000"
},
{
"name": "X-Frame-Options",
"value": "SAMEORIGIN"
},
{
"name": "X-XSS-Protection",
"value": "0"
}
],
"cookies": [
{
"name": "1P_JAR",
"value": "2024-01-05-20",
"path": "/",
"domain": ".google.com",
"expires": "2024-02-04T20:42:08.000Z",
"httpOnly": false,
"secure": true,
"sameSite": "none"
},
{
"name": "OTZ",
"value": "",
"path": "/",
"domain": "www.google.com",
"expires": "1990-01-01T00:00:00.000Z",
"httpOnly": false,
"secure": false
},
{
"name": "OTZ",
"value": "",
"path": "/",
"domain": ".www.google.com",
"expires": "1990-01-01T00:00:00.000Z",
"httpOnly": false,
"secure": false
},
{
"name": "OTZ",
"value": "",
"path": "/",
"domain": "google.com",
"expires": "1990-01-01T00:00:00.000Z",
"httpOnly": false,
"secure": false
},
{
"name": "OTZ",
"value": "",
"path": "/",
"domain": ".google.com",
"expires": "1990-01-01T00:00:00.000Z",
"httpOnly": false,
"secure": false
},
{
"name": "AEC",
"value": "Ackid1QwcPoX9Lo...Qv2Qn6_kbwSMZabs1Dmjgjhcbcs",
"path": "/",
"domain": ".google.com",
"expires": "2024-05-07T17:19:43.000Z",
"httpOnly": true,
"secure": true,
"sameSite": "lax"
},
{
"name": "NID",
"value": "511=SyoR8m7-p65I5vBw...7hFjR54yfx_1mZR5",
"path": "/",
"domain": ".google.com",
"expires": "2024-07-06T20:32:24.000Z",
"httpOnly": true,
"secure": true,
"sameSite": "none"
},
{
"name": "SIDCC",
"value": "ABTWhQFYVGxbJnkXhg...Z1goPb2KD6p4WQIts6A",
"path": "/",
"domain": ".google.com",
"expires": "2025-01-04T20:42:08.000Z",
"httpOnly": false,
"secure": false
}
],
"content": {
"size": 0,
"mimeType": "text/html",
"compression": 2019
},
"redirectURL": "",
"headersSize": 3329,
"bodySize": -2019,
"_transferSize": 1310,
"_error": null
},
"serverIPAddress": "142.250.105.102",
"startedDateTime": "2024-01-05T20:42:08.414Z",
"time": 411.3230001655817,
"timings": {
"blocked": 262.8310001701266,
"dns": 0.006000000000000227,
"ssl": 13.739999999999995,
"connect": 26.442999999999998,
"send": 0.2639999999999958,
"wait": 119.58700006829203,
"receive": 2.191999927163124,
"_blocked_queueing": 206.57200017012656,
"_blocked_proxy": 26.493
}
}
]
}
}