implement org.apache HttpClient

Author: drm

lib/commons-codec-1.9.jar

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+ROOT=$(cd $(dirname $0) && pwd)
+
+fetch() {
+	fetch_maven_deps() {
+		local dir=$(mktemp -d);
+
+		(
+			cd "$dir";
+			cat > pom.xml <<-EOF
+				<project>
+					<modelVersion>4.0.0</modelVersion>
+					<groupId>temp</groupId>
+					<artifactId>temp</artifactId>
+					<version>master</version>
+					<repositories>
+						<repository>
+							<id>jitpack.io</id>
+							<url>https://jitpack.io</url>
+						</repository>
+					</repositories>
+					<dependencies>
+			EOF
+
+			for package in "$@"; do
+				echo "$package" | \
+					sed -E 's!([^:]+):([^:]+):([^:]+)!<dependency><groupId>\1</groupId><artifactId>\2</artifactId><version>\3</version></dependency>!g' \
+					>> pom.xml
+			done;
+
+			cat >> pom.xml <<-EOF
+					</dependencies>
+				</project>
+			EOF
+
+			mvn dependency:copy-dependencies
+			mv target/dependency/* "$ROOT";
+		)
+		rm -rf "$dir";
+	}
+	
+	fetch_maven_deps \
+		'org.apache.httpcomponents:httpclient:4.5.3' \
+		'junit:junit:4.12' \
+		'org.slf4j:slf4j-log4j12:1.7.7' \
+		'org.jsoup:jsoup:1.11.3'
+
+	curl -sL https://github.com/drm/java-redis-client/releases/download/v2.0.2/java-redis-client-v2.0.2--javac-11.0.2.jar -o ./java-redis-client-v2.0.2.jar
+	curl -sL https://github.com/drm/java-redis-collections/releases/download/v.0.3.0-beta.3/java-redis-collections-v.0.3.0-beta.3--javac-11.0.3.jar -o ./java-redis-collections-v0.3.0-beta.3.jar
+}
+
+clean() {
+	rm -f *.jar;
+}
+
+if [[ "$1" == "" ]]; then
+	echo "Usage: ${0} [clean] fetch"
+	exit 1
+fi
+
+set -x
+set -e
+
+
+for a in $@; do
+	$a;
+done

lib/fetch.sh

@@ -2,6 +2,12 @@
 
 import nl.melp.redis.Redis;
 import nl.melp.redis.collections.*;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
@@ -17,12 +23,8 @@
 import java.net.ConnectException;
 import java.net.Socket;
 import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
 import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
-import java.time.Duration;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
@@ -35,15 +37,15 @@ public class LinkChecker {
 	private static Logger logger = LoggerFactory.getLogger(LinkChecker.class);
 	private static int timeout = 30;
 
-	private final BlockingDeque<HttpClient> clients;
+	private final BlockingDeque<CloseableHttpClient> clients;
 	private final Map<URI, Integer> statuses;
 	private final Collection<URI> urls;
 	private final Map<URI, Set<URI>> reverseLinks;
 	private final Map<String, Set<URI>> invalidUrls;
 	private final ExecutorService executor;
 	private final Set<Future> running;
 	private final BiPredicate<URI, URI> shouldFollowLinks;
-	private final BiPredicate<URI, HttpResponse<String>> shouldExtractLinks;
+	private final BiPredicate<URI, HttpEntity> shouldExtractLinks;
 	private final int msDelay;
 	private long startTimeMs;
 
@@ -53,7 +55,7 @@ public LinkChecker(
 		Map<URI, Set<URI>> reverseLinks,
 		Map<String, Set<URI>> invalidUrls,
 		BiPredicate<URI, URI> shouldFollowLinks,
-		BiPredicate<URI, HttpResponse<String>> shouldExtractLinks,
+		BiPredicate<URI, HttpEntity> shouldExtractLinks,
 		int numThreads,
 		int msDelay
 	) {
@@ -69,7 +71,7 @@ public LinkChecker(
 		this.msDelay = msDelay;
 
 		for (int i = 0; i < numThreads; i++) {
-			clients.offer(HttpClient.newHttpClient());
+			clients.offer(HttpClients.createMinimal());
 		}
 	}
 
@@ -133,7 +135,7 @@ public Map<URI, Integer> run() throws InterruptedException {
 					continue;
 				}
 				statuses.put(url, -1);
-				HttpClient httpClient = clients.take();
+				CloseableHttpClient httpClient = clients.take();
 
 				Future<?> task = executor.submit(
 					() -> {
@@ -147,42 +149,42 @@ public Map<URI, Integer> run() throws InterruptedException {
 							}
 
 							logger.trace("OPENING " + url);
-							HttpRequest request = HttpRequest.newBuilder()
-								.uri(url)
-								.timeout(Duration.ofSeconds(timeout))
-								.build();
-
-							HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
-							int status = response.statusCode();
-							statuses.put(url, status);
-
-							if (status >= 400) {
-								logger.info("Got status " + status + " at " + url + "; so far referred to by " + reverseLinks.get(url));
-							} else {
-								logger.trace("Got status " + status + " at " + url);
-							}
 
-							if (shouldExtractLinks.test(url, response)) {
-								String contentType = response.headers().firstValue("Content-Type").orElse("");
-								if (status == 200) {
-									if (contentType.startsWith("text/html")) {
-										response.body();
-										Document d = Jsoup.parse(response.body());
-										Elements links = d.select("a[href]");
-										logger.trace("Found " + links.size() + " on " + url);
-										for (Element link : links) {
-											addUrl(url, link.attr("href"));
+							var request = new HttpGet(url);
+							try (CloseableHttpResponse response = httpClient.execute(request)) {
+								int status = response.getStatusLine().getStatusCode();
+								statuses.put(url, status);
+
+								if (status >= 400) {
+									logger.info("Got status " + status + " at " + url + "; so far referred to by " + reverseLinks.get(url));
+								} else {
+									logger.trace("Got status " + status + " at " + url);
+								}
+								HttpEntity responseEntity = response.getEntity();
+
+								if (shouldExtractLinks.test(url, responseEntity)) {
+									Header contentTypeHeader = response.getFirstHeader("Content-Type");
+									String contentType = contentTypeHeader == null ? "UNKNOWN" : contentTypeHeader.getValue();
+
+									if (status == 200) {
+										if (contentType.startsWith("text/html")) {
+											Document d = Jsoup.parse(responseEntity.getContent(), "UTF-8", url.toString());
+											Elements links = d.select("a[href]");
+											logger.trace("Found " + links.size() + " on " + url);
+											for (Element link : links) {
+												addUrl(url, link.attr("href"));
+											}
+										} else {
+											logger.trace("Not following links in content type " + contentType);
+										}
+									} else if (response.getFirstHeader("Location") != null) {
+										String location = response.getFirstHeader("Location").getValue();
+										if (addUrl(url, location)) {
+											logger.trace("Following redirect (" + status + ") [" + url + " => " + location);
 										}
 									} else {
-										logger.trace("Not following links in content type " + response.headers().firstValue("Content-Type").orElse("UNKNOWN"));
-									}
-								} else if (response.headers().firstValue("Location").isPresent()) {
-									String location = response.headers().firstValue("Location").get();
-									if (addUrl(url, location)) {
-										logger.trace("Following redirect (" + status + ") [" + url + " => " + location);
+										logger.debug("Skipping {}, content-type: {}", url, contentType);
 									}
-								} else {
-									logger.debug("Skipping {}, content-type: {}", url, contentType);
 								}
 							}
 						} catch (java.lang.IllegalArgumentException | IOException e) {