* Handle proxy configuration, closes #5476 * typos
This commit is contained in:
parent
76152500a5
commit
89e6dbe2fe
|
@ -8,6 +8,16 @@
|
|||
|
||||
# Use system defined HTTP proxies
|
||||
-Djava.net.useSystemProxies=true
|
||||
# You should probably define the properties below, if they are not already
|
||||
# defined in the environment variable JAVA_TOOL_OPTIONS.
|
||||
# Refer to the Java documentation (for example, at
|
||||
# https://docs.oracle.com/javase/8/docs/api/java/net/doc-files/net-properties.html
|
||||
# for the precise meaning of each option)
|
||||
#-Dhttp.proxyHost=myproxy.mydomain.org
|
||||
#-Dhttp.proxyPort=8080
|
||||
#-Dhttps.proxyHost=myproxy.mydomain.org
|
||||
#-Dhttps.proxyPort=8080
|
||||
#-Dhttp.nonProxyHosts="*.mydomain.org|localhost|127.*"
|
||||
|
||||
#-XX:+UseLargePages
|
||||
#-Dsomevar="%SOMEVAR%"
|
||||
|
|
|
@ -35,6 +35,8 @@ import java.nio.charset.StandardCharsets;
|
|||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
import org.apache.hc.client5.http.ClientProtocolException;
|
||||
import org.apache.hc.client5.http.classic.methods.HttpGet;
|
||||
|
@ -83,6 +85,7 @@ public class HttpClient {
|
|||
private HttpHost proxy;
|
||||
private int proxyPort;
|
||||
private String proxyHost;
|
||||
private Pattern nonProxyHosts;
|
||||
private DefaultProxyRoutePlanner routePlanner;
|
||||
|
||||
public HttpClient() {
|
||||
|
@ -138,21 +141,39 @@ public class HttpClient {
|
|||
}
|
||||
});
|
||||
|
||||
if (System.getProperty("http.proxyHost") != null) {
|
||||
proxyHost = System.getProperty("http.proxyHost");
|
||||
}
|
||||
|
||||
if (System.getProperty("http.proxyPort") != null) {
|
||||
proxyPort = Integer.parseInt(System.getProperty("http.proxyPort"));
|
||||
// Should we use the system defined proxy
|
||||
if ("true".equals(System.getProperty("java.net.useSystemProxies"))) {
|
||||
logger.info("Use system defined proxy for http connections");
|
||||
httpClient = httpClientBuilder.useSystemProperties().build();
|
||||
return;
|
||||
}
|
||||
|
||||
// Is a proxy defined
|
||||
proxyHost = System.getProperty("http.proxyHost");
|
||||
proxyPort = Integer.parseInt(System.getProperty("http.proxyPort", "0"));
|
||||
if (proxyHost != null && proxyPort != 0) {
|
||||
proxy = new HttpHost("http", proxyHost, proxyPort);
|
||||
httpClientBuilder.setProxy(proxy);
|
||||
}
|
||||
if (proxy != null) {
|
||||
routePlanner = new DefaultProxyRoutePlanner(proxy);
|
||||
httpClientBuilder.setRoutePlanner(routePlanner);
|
||||
logger.info("Use provided proxy " + proxy.toString() + " for http connections");
|
||||
String strNonProxyHosts = System.getProperty("http.nonProxyHosts");
|
||||
nonProxyHosts = fromHostsToPattern(strNonProxyHosts);
|
||||
if (nonProxyHosts != null) {
|
||||
logger.info("except for hosts matching " + strNonProxyHosts);
|
||||
}
|
||||
if (proxy != null) {
|
||||
// Manage nonProxyHosts
|
||||
routePlanner = new DefaultProxyRoutePlanner(proxy) {
|
||||
|
||||
@Override
|
||||
protected HttpHost determineProxy(HttpHost target, HttpContext context) throws HttpException {
|
||||
String host = target.getHostName();
|
||||
if (nonProxyHosts != null && nonProxyHosts.matcher(host).matches()) {
|
||||
return null;
|
||||
}
|
||||
return proxy;
|
||||
}
|
||||
};
|
||||
httpClientBuilder.setRoutePlanner(routePlanner);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Placeholder for future Basic Auth implementation
|
||||
|
@ -173,6 +194,28 @@ public class HttpClient {
|
|||
httpClient = httpClientBuilder.build();
|
||||
}
|
||||
|
||||
protected static Pattern fromHostsToPattern(final String hostsList) {
|
||||
if (hostsList == null) {
|
||||
return null;
|
||||
}
|
||||
String[] hosts = hostsList.split("\\|");
|
||||
String[] rHosts = new String[hosts.length];
|
||||
// Transform glob to regex using Pattern.quote() to avoid regex injections
|
||||
for (int i = 0; i < hosts.length; i++) {
|
||||
String p = hosts[i];
|
||||
if (p.startsWith("*") && p.endsWith("*")) {
|
||||
rHosts[i] = ".*" + Pattern.quote(p.substring(1, p.length() - 2)) + ".*";
|
||||
} else if (p.startsWith("*")) {
|
||||
rHosts[i] = ".*" + Pattern.quote(p.substring(1));
|
||||
} else if (p.endsWith("*")) {
|
||||
rHosts[i] = Pattern.quote(p.substring(0, p.length() - 1)) + ".*";
|
||||
} else {
|
||||
rHosts[i] = Pattern.quote(p);
|
||||
}
|
||||
}
|
||||
return Pattern.compile(String.join("|", rHosts));
|
||||
}
|
||||
|
||||
public String getAsString(String urlString, Header[] headers) throws IOException {
|
||||
|
||||
final HttpClientResponseHandler<String> responseHandler = new HttpClientResponseHandler<String>() {
|
||||
|
|
|
@ -0,0 +1,31 @@
|
|||
|
||||
package com.google.refine.util;
|
||||
|
||||
import org.testng.Assert;
|
||||
import org.testng.annotations.Test;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.Matcher;
|
||||
|
||||
public class HttpClientTests {
|
||||
|
||||
@Test
|
||||
public void fromHostsToPattern() {
|
||||
String strPattern1 = "localhost|127.0.0.1";
|
||||
Pattern pattern1 = HttpClient.fromHostsToPattern(strPattern1);
|
||||
Assert.assertTrue(pattern1.matcher("localhost").matches());
|
||||
Assert.assertFalse(pattern1.matcher("host.domain.org").matches());
|
||||
|
||||
String strPattern2 = "*.domain.org";
|
||||
Pattern pattern2 = HttpClient.fromHostsToPattern(strPattern2);
|
||||
Assert.assertFalse(pattern2.matcher("localhost").matches());
|
||||
Assert.assertTrue(pattern2.matcher("host.domain.org").matches());
|
||||
|
||||
String strPattern3 = "*.domain.org|*.any.*|myhosts.*";
|
||||
Pattern pattern3 = HttpClient.fromHostsToPattern(strPattern3);
|
||||
Assert.assertFalse(pattern3.matcher("localhost").matches());
|
||||
Assert.assertTrue(pattern3.matcher("host.domain.org").matches());
|
||||
Assert.assertTrue(pattern3.matcher("random.domain.any.com").matches());
|
||||
Assert.assertTrue(pattern3.matcher("myhosts.mydomain.mine").matches());
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue