diff --git a/CLA-rev2-digital.txt b/CLA-rev2-digital.txt index 30eeba02..427834a8 100644 --- a/CLA-rev2-digital.txt +++ b/CLA-rev2-digital.txt @@ -25,4 +25,6 @@ Vitali Kepin https://github.com/vkepin 2017-05-16 D. Hollingsworth https://github.com/hollingsworthd 2017-06-25 Nicholas DiPiazza https://github.com/nddipiazza/ 2017-08-08 Alena Liashenka https://github.com/eichhornchen27 2017-12-08 -Uladzislau Arlouski https://github.com/uarlouski 2017-12-11 \ No newline at end of file +Uladzislau Arlouski https://github.com/uarlouski 2017-12-11 +contributions by (Vincent Smith https://github.com/vosmith 2018-06-08) +included by (Adam Howard https://github.com/medavox 2018-07-06) \ No newline at end of file diff --git a/src/com/machinepublishers/jbrowserdriver/Settings.java b/src/com/machinepublishers/jbrowserdriver/Settings.java index 3a095c0b..ad0394f3 100644 --- a/src/com/machinepublishers/jbrowserdriver/Settings.java +++ b/src/com/machinepublishers/jbrowserdriver/Settings.java @@ -160,6 +160,7 @@ private enum PropertyName { USER_DATA_DIRECTORY("jbd.userdatadirectory"), CSRF_REQUEST_TOKEN("jbd.csrfreqtoken"), CSRF_RESPONSE_TOKEN("jbd.csrfresptoken"), + BLOCK_MEDIA("jbd.blockmedia"), @Deprecated WIRE_CONSOLE("jbd.wireconsole"), @Deprecated @@ -226,6 +227,7 @@ public static class Builder { private String csrfRequestToken; private String csrfResponseToken; private InetAddress nicAddress; + private boolean blockMedia; /** * Headers to be sent on each request. @@ -1134,7 +1136,7 @@ public Builder javaExportModules(boolean javaExportModules) { *
  • Java system property jbd.userdatadirectory overrides this setting. Use a string file path as the value.
    • *
  • {@link Capabilities} name jbd.userdatadirectory alternately configures this setting. Use a string file path as the value.
    • *

    - * + * * @param userDataDirectory * A directory to store user website data, e.g. localStorage. * @return this Builder @@ -1148,7 +1150,7 @@ public Builder userDataDirectory(File userDataDirectory) { /** * Used for binding to a specific NIC - * + * * @param nicAddress * @return this Builder */ @@ -1159,7 +1161,7 @@ public Builder localIp(InetAddress nicAddress) { /** * Enables CSRF token handling. Searches for XSRF-TOKEN in response headers and sends X-XSRF-TOKEN in request headers. - * + * * @return this Builder */ public Builder csrf() { @@ -1168,7 +1170,7 @@ public Builder csrf() { /** * Enables CSRF token handling - * + * * @param requestToken * The header to send in each request header * @param responseToken @@ -1181,6 +1183,26 @@ public Builder csrf(String requestToken, String responseToken) { return this; } + /** + * Whether requests for common media types should be blocked. + *

    + * Based on the content-type or file extension of the requested asset + *

    + * Defaults to false. + * + *

    + * + * @param blockMedia + * @return this Builder + */ + public Builder blockMedia(boolean blockMedia) { + this.blockMedia = blockMedia; + return this; + } + /** * @deprecated Will be removed in v2.0.0. Instead use Settings Builder's logWire, logsMax, or logger. */ @@ -1306,6 +1328,7 @@ public Capabilities buildCapabilities() { set(capabilities, PropertyName.PROXY_PASSWORD, proxy.password()); set(capabilities, PropertyName.PROXY_EXPECT_CONTINUE, proxy.expectContinue()); } + set(capabilities, PropertyName.BLOCK_MEDIA, this.blockMedia); return capabilities; } @@ -1494,6 +1517,7 @@ private static File parse(Map capabilities, PropertyName name, File fallback) { private final String csrfRequestToken; private final String csrfResponseToken; private final InetAddress nicAddress; + private final boolean blockMedia; private Settings(Settings.Builder builder, Map properties) { Settings.Builder defaults = Settings.builder(); @@ -1682,6 +1706,7 @@ private Settings(Settings.Builder builder, Map properties) { scriptBuilder.append("'));})();"); scriptBuilder.append(""); this.script = scriptBuilder.toString(); + this.blockMedia = parse(properties, PropertyName.BLOCK_MEDIA, builder.blockMedia); } RequestHeaders headers() { @@ -1859,4 +1884,8 @@ String getCsrfResponseToken() { InetAddress getLocalIp() { return nicAddress; } + + boolean blockMedia() { + return blockMedia; + } } diff --git a/src/com/machinepublishers/jbrowserdriver/StreamConnection.java b/src/com/machinepublishers/jbrowserdriver/StreamConnection.java index a344a842..3b33077b 100644 --- a/src/com/machinepublishers/jbrowserdriver/StreamConnection.java +++ b/src/com/machinepublishers/jbrowserdriver/StreamConnection.java @@ -100,7 +100,7 @@ class StreamConnection extends HttpURLConnection implements Closeable { ".svg", ".gif", ".jpeg", ".jpg", ".png", ".ico", ".webm", ".mp4", ".ogg", ".ogv", ".mp3", ".aac", ".wav", ".swf", ".woff", - ".otf", ".ttf" }))); + ".otf", ".ttf", ".css" }))); private final Map> reqHeaders = new LinkedHashMap>(); private final Map reqHeadersCasing = new HashMap(); @@ -263,54 +263,58 @@ public void connect() throws IOException { } else if (isBlocked(url.getHost())) { skip.set(true); } else if (SettingsManager.settings() != null) { - config.get() - .setCookieSpec("custom") - .setSocketTimeout(SettingsManager.settings().socketTimeout()) - .setConnectTimeout(SettingsManager.settings().connectTimeout()) - .setConnectionRequestTimeout(SettingsManager.settings().connectionReqTimeout()) - .setLocalAddress(SettingsManager.settings().getLocalIp()); - URI uri = null; - try { - uri = url.toURI(); - } catch (URISyntaxException e) { - //decode components of the url first, because often the problem is partially encoded urls - uri = new URI(url.getProtocol(), - url.getAuthority(), - url.getPath() == null ? null : URLDecoder.decode(url.getPath(), "utf-8"), - url.getQuery() == null ? null : URLDecoder.decode(url.getQuery(), "utf-8"), - url.getRef() == null ? null : URLDecoder.decode(url.getRef(), "utf-8")); - } - if ("OPTIONS".equals(method.get())) { - req.set(new HttpOptions(uri)); - } else if ("GET".equals(method.get())) { - req.set(new HttpGet(uri)); - } else if ("HEAD".equals(method.get())) { - req.set(new HttpHead(uri)); - } else if ("POST".equals(method.get())) { - req.set(new HttpPost(uri)); - } else if ("PUT".equals(method.get())) { - req.set(new HttpPut(uri)); - } else if ("DELETE".equals(method.get())) { - req.set(new HttpDelete(uri)); - } else if ("TRACE".equals(method.get())) { - req.set(new HttpTrace(uri)); - } else if ("PATCH".equals(method.get())) { - req.set(new HttpPatch(uri)); - } - processHeaders(SettingsManager.settings(), req.get()); - ProxyConfig proxy = SettingsManager.settings().proxy(); - if (proxy != null && !proxy.directConnection() && !proxy.nonProxyHosts().contains(uri.getHost())) { - config.get().setExpectContinueEnabled(proxy.expectContinue()); - InetSocketAddress proxyAddress = new InetSocketAddress(proxy.host(), proxy.port()); - if (proxy.type() == ProxyConfig.Type.SOCKS) { - context.get().setAttribute("proxy.socks.address", proxyAddress); - } else { - config.get().setProxy(new HttpHost(proxy.host(), proxy.port())); + if (SettingsManager.settings().blockMedia() && isMedia()) { + skip.set(true); + } else { + config.get() + .setCookieSpec("custom") + .setSocketTimeout(SettingsManager.settings().socketTimeout()) + .setConnectTimeout(SettingsManager.settings().connectTimeout()) + .setConnectionRequestTimeout(SettingsManager.settings().connectionReqTimeout()) + .setLocalAddress(SettingsManager.settings().getLocalIp()); + URI uri = null; + try { + uri = url.toURI(); + } catch (URISyntaxException e) { + //decode components of the url first, because often the problem is partially encoded urls + uri = new URI(url.getProtocol(), + url.getAuthority(), + url.getPath() == null ? null : URLDecoder.decode(url.getPath(), "utf-8"), + url.getQuery() == null ? null : URLDecoder.decode(url.getQuery(), "utf-8"), + url.getRef() == null ? null : URLDecoder.decode(url.getRef(), "utf-8")); + } + if ("OPTIONS".equals(method.get())) { + req.set(new HttpOptions(uri)); + } else if ("GET".equals(method.get())) { + req.set(new HttpGet(uri)); + } else if ("HEAD".equals(method.get())) { + req.set(new HttpHead(uri)); + } else if ("POST".equals(method.get())) { + req.set(new HttpPost(uri)); + } else if ("PUT".equals(method.get())) { + req.set(new HttpPut(uri)); + } else if ("DELETE".equals(method.get())) { + req.set(new HttpDelete(uri)); + } else if ("TRACE".equals(method.get())) { + req.set(new HttpTrace(uri)); + } else if ("PATCH".equals(method.get())) { + req.set(new HttpPatch(uri)); + } + processHeaders(SettingsManager.settings(), req.get()); + ProxyConfig proxy = SettingsManager.settings().proxy(); + if (proxy != null && !proxy.directConnection() && !proxy.nonProxyHosts().contains(uri.getHost())) { + config.get().setExpectContinueEnabled(proxy.expectContinue()); + InetSocketAddress proxyAddress = new InetSocketAddress(proxy.host(), proxy.port()); + if (proxy.type() == ProxyConfig.Type.SOCKS) { + context.get().setAttribute("proxy.socks.address", proxyAddress); + } else { + config.get().setProxy(new HttpHost(proxy.host(), proxy.port())); + } } + context.get().setCookieStore(cookieStore); + context.get().setRequestConfig(config.get().build()); + StatusMonitor.instance().monitor(url, this); } - context.get().setCookieStore(cookieStore); - context.get().setRequestConfig(config.get().build()); - StatusMonitor.instance().monitor(url, this); } } } catch (Throwable t) { diff --git a/src/com/machinepublishers/jbrowserdriver/ad-hosts.txt b/src/com/machinepublishers/jbrowserdriver/ad-hosts.txt index f58eb59e..d8a9b792 100644 --- a/src/com/machinepublishers/jbrowserdriver/ad-hosts.txt +++ b/src/com/machinepublishers/jbrowserdriver/ad-hosts.txt @@ -28641,4 +28641,4 @@ z.times.lv ztrack.net zu-yuan.com zwierzu.zxy.me -z.zeroredirect2.com \ No newline at end of file +z.zeroredirect2.com