Handle proxy configuration, closes #5476 (#5477)

* Handle proxy configuration, closes #5476

* typos
This commit is contained in:
tledoux 2023-02-09 11:39:59 +01:00 committed by GitHub
parent 76152500a5
commit 89e6dbe2fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 95 additions and 11 deletions

View File

@ -8,6 +8,16 @@
# Use system defined HTTP proxies
-Djava.net.useSystemProxies=true
# You should probably define the properties below, if they are not already
# defined in the environment variable JAVA_TOOL_OPTIONS.
# Refer to the Java documentation (for example, at
# https://docs.oracle.com/javase/8/docs/api/java/net/doc-files/net-properties.html
# for the precise meaning of each option)
#-Dhttp.proxyHost=myproxy.mydomain.org
#-Dhttp.proxyPort=8080
#-Dhttps.proxyHost=myproxy.mydomain.org
#-Dhttps.proxyPort=8080
#-Dhttp.nonProxyHosts="*.mydomain.org|localhost|127.*"
#-XX:+UseLargePages
#-Dsomevar="%SOMEVAR%"

View File

@ -35,6 +35,8 @@ import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import org.apache.hc.client5.http.ClientProtocolException;
import org.apache.hc.client5.http.classic.methods.HttpGet;
@ -83,6 +85,7 @@ public class HttpClient {
private HttpHost proxy;
private int proxyPort;
private String proxyHost;
private Pattern nonProxyHosts;
private DefaultProxyRoutePlanner routePlanner;
public HttpClient() {
@ -138,21 +141,39 @@ public class HttpClient {
}
});
if (System.getProperty("http.proxyHost") != null) {
proxyHost = System.getProperty("http.proxyHost");
}
if (System.getProperty("http.proxyPort") != null) {
proxyPort = Integer.parseInt(System.getProperty("http.proxyPort"));
// Should we use the system defined proxy
if ("true".equals(System.getProperty("java.net.useSystemProxies"))) {
logger.info("Use system defined proxy for http connections");
httpClient = httpClientBuilder.useSystemProperties().build();
return;
}
// Is a proxy defined
proxyHost = System.getProperty("http.proxyHost");
proxyPort = Integer.parseInt(System.getProperty("http.proxyPort", "0"));
if (proxyHost != null && proxyPort != 0) {
proxy = new HttpHost("http", proxyHost, proxyPort);
httpClientBuilder.setProxy(proxy);
}
if (proxy != null) {
routePlanner = new DefaultProxyRoutePlanner(proxy);
httpClientBuilder.setRoutePlanner(routePlanner);
logger.info("Use provided proxy " + proxy.toString() + " for http connections");
String strNonProxyHosts = System.getProperty("http.nonProxyHosts");
nonProxyHosts = fromHostsToPattern(strNonProxyHosts);
if (nonProxyHosts != null) {
logger.info("except for hosts matching " + strNonProxyHosts);
}
if (proxy != null) {
// Manage nonProxyHosts
routePlanner = new DefaultProxyRoutePlanner(proxy) {
@Override
protected HttpHost determineProxy(HttpHost target, HttpContext context) throws HttpException {
String host = target.getHostName();
if (nonProxyHosts != null && nonProxyHosts.matcher(host).matches()) {
return null;
}
return proxy;
}
};
httpClientBuilder.setRoutePlanner(routePlanner);
}
}
// TODO: Placeholder for future Basic Auth implementation
@ -173,6 +194,28 @@ public class HttpClient {
httpClient = httpClientBuilder.build();
}
protected static Pattern fromHostsToPattern(final String hostsList) {
if (hostsList == null) {
return null;
}
String[] hosts = hostsList.split("\\|");
String[] rHosts = new String[hosts.length];
// Transform glob to regex using Pattern.quote() to avoid regex injections
for (int i = 0; i < hosts.length; i++) {
String p = hosts[i];
if (p.startsWith("*") && p.endsWith("*")) {
rHosts[i] = ".*" + Pattern.quote(p.substring(1, p.length() - 2)) + ".*";
} else if (p.startsWith("*")) {
rHosts[i] = ".*" + Pattern.quote(p.substring(1));
} else if (p.endsWith("*")) {
rHosts[i] = Pattern.quote(p.substring(0, p.length() - 1)) + ".*";
} else {
rHosts[i] = Pattern.quote(p);
}
}
return Pattern.compile(String.join("|", rHosts));
}
public String getAsString(String urlString, Header[] headers) throws IOException {
final HttpClientResponseHandler<String> responseHandler = new HttpClientResponseHandler<String>() {

View File

@ -0,0 +1,31 @@
package com.google.refine.util;
import org.testng.Assert;
import org.testng.annotations.Test;
import java.time.OffsetDateTime;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
public class HttpClientTests {
@Test
public void fromHostsToPattern() {
String strPattern1 = "localhost|127.0.0.1";
Pattern pattern1 = HttpClient.fromHostsToPattern(strPattern1);
Assert.assertTrue(pattern1.matcher("localhost").matches());
Assert.assertFalse(pattern1.matcher("host.domain.org").matches());
String strPattern2 = "*.domain.org";
Pattern pattern2 = HttpClient.fromHostsToPattern(strPattern2);
Assert.assertFalse(pattern2.matcher("localhost").matches());
Assert.assertTrue(pattern2.matcher("host.domain.org").matches());
String strPattern3 = "*.domain.org|*.any.*|myhosts.*";
Pattern pattern3 = HttpClient.fromHostsToPattern(strPattern3);
Assert.assertFalse(pattern3.matcher("localhost").matches());
Assert.assertTrue(pattern3.matcher("host.domain.org").matches());
Assert.assertTrue(pattern3.matcher("random.domain.any.com").matches());
Assert.assertTrue(pattern3.matcher("myhosts.mydomain.mine").matches());
}
}