refactor in separate classes, add tests, fix recheck

Author: drm

README.md

@@ -40,8 +40,7 @@ java -cp 'lib/*:bin/*.jar' nl.melp.linkchecker.LinkChecker
 | `--follow-local` | Only local links to that local* domain are followed |
 | `--follow-from-local` | Only follow links that are mentioned on the local* domain. This means that the link checker only spans over multiple hosts *once*. |
 | `--no-follow` | No links are followed. This is typically useful in combination with the `--recheck` flag |
-| `--recheck` | Reset the status for each of the previously failed URLs, and try them again. |
-| `--recheck-only-errors` | Only recheck links that had an internal error state, i.e. all urls that had connection errors, timeouts, etc. |
+| `--recheck` | Reset the status for each of the previously failed URLs, and recheck the pages they are mentioned on. |
 | `--no-recheck` | Don't do recheck, even if url's are marked as "processing". |
 | `--reset` | Start with a clean slate |
 | `--resume` | Resume a previously stopped session. |
@@ -76,4 +75,4 @@ won't need further configuration.
 Please report them at github.com/drm/java-linkchecker
 
 # Copyright
-(c) 2019 Gerard van Helden
+(c) 2019-2020 Gerard van Helden

lib/fetch.sh

@@ -46,7 +46,7 @@ fetch() {
 		'org.slf4j:slf4j-log4j12:1.7.7' \
 		'org.jsoup:jsoup:1.11.3'
 
-	curl -sL https://github.com/drm/java-redis-client/releases/download/v2.0.2/java-redis-client-v2.0.2--javac-11.0.2.jar -o ./java-redis-client-v2.0.2.jar
+	curl -sL https://github.com/drm/java-redis-client/releases/download/v2.1.0/java-redis-client-v2.1.0--javac-11.0.9.1.jar -o ./java-redis-client-v2.1.0.jar
 	curl -sL https://github.com/drm/java-redis-collections/releases/download/v1.0.2/java-redis-collections-v1.0.2--javac-11.0.7.jar -o ./java-redis-collections-v1.0.2.jar
 }
 

lib/java-redis-client-v2.0.2.jar

@@ -17,5 +17,5 @@ log4j.appender.fileName.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
 log4j.rootLogger=info, stdout, fileName
 
 org.apache.http=info
-log4j.logger.nl.melp.linkchecker=debug
+log4j.logger.nl.melp.linkchecker=info
 

lib/java-redis-client-v2.1.0.jar

@@ -0,0 +1,12 @@
+package nl.melp.linkchecker;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Set;
+
+public interface Extractor {
+	Set<String> extract(URI url, int statusCode, CloseableHttpResponse response, HttpEntity responseEntity) throws IOException;
+}

lib/java-redis-collections-v1.0.2.jar

@@ -0,0 +1,96 @@
+package nl.melp.linkchecker;
+
+import nl.melp.linkchecker.URIResolver.InvalidURIException;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+public class Fetcher {
+	private static final int timeout = 30;
+
+	private static final RequestConfig requestConfig = RequestConfig.custom()
+		.setConnectTimeout(timeout * 1000)
+		.setConnectionRequestTimeout(timeout * 1000)
+		.setSocketTimeout(timeout * 1000)
+		.build();
+
+	public static class Result {
+		private final URI uri;
+		private final int statusCode;
+		private final Set<URI> referredLinks;
+		private final Set<String> invalidLinks;
+
+		public Result(URI uri, int statusCode, Set<URI> referredLinks, Set<String> invalidLinks) {
+			this.uri = uri;
+			this.statusCode = statusCode;
+			this.referredLinks = referredLinks;
+			this.invalidLinks = invalidLinks;
+		}
+
+		public URI getUri() {
+			return uri;
+		}
+
+		public int getStatusCode() {
+			return statusCode;
+		}
+
+		public Set<URI> getReferredLinks() {
+			return referredLinks;
+		}
+
+		public Set<String> getInvalidLinks() {
+			return invalidLinks;
+		}
+	}
+
+	private final Logger logger;
+	private final RunConfig config;
+	private final Extractor extractor;
+	private final URIResolver resolver;
+
+	public Fetcher(Logger logger, RunConfig config, Extractor extractor, URIResolver resolver) {
+		this.logger = logger;
+		this.config = config;
+		this.extractor = extractor;
+		this.resolver = resolver;
+	}
+
+	public Result fetch(CloseableHttpClient httpClient, URI url) {
+		var request = new HttpGet(url);
+		request.setConfig(requestConfig);
+		try (CloseableHttpResponse response = httpClient.execute(request)) {
+			int statusCode = response.getStatusLine().getStatusCode();
+
+			logger.trace("Got status " + statusCode + " at " + url);
+
+			HttpEntity responseEntity = response.getEntity();
+			Set<URI> links = new LinkedHashSet<>();
+			Set<String> invalidLinks = new LinkedHashSet<>();
+
+			if (config.shouldExtractLinks(url)) {
+				for (String link : extractor.extract(url, statusCode, response, responseEntity)) {
+					try {
+						final URI target = resolver.resolveUri(url, link);
+						if (target != null) {
+							links.add(target);
+						}
+					} catch (InvalidURIException e) {
+						invalidLinks.add(link);
+					}
+				}
+			}
+			return new Result(url, statusCode, links, invalidLinks);
+		} catch (IOException e) {
+			return new Result(url, 0, null, null);
+		}
+	}
+}

src/log4j.properties

@@ -0,0 +1,52 @@
+package nl.melp.linkchecker;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+public class HtmlExtractor implements Extractor {
+	private final Logger logger;
+
+	public HtmlExtractor(Logger logger) {
+		this.logger = logger;
+	}
+
+	@Override
+	public Set<String> extract(URI url, int statusCode, CloseableHttpResponse response, HttpEntity responseEntity) throws IOException {
+		Set<String> referred = new LinkedHashSet<>();
+		Header contentTypeHeader = response.getFirstHeader("Content-Type");
+		String contentType = contentTypeHeader == null ? "UNKNOWN" : contentTypeHeader.getValue();
+
+		if (statusCode == 200) {
+			if (contentType.startsWith("text/html")) {
+				Document d = Jsoup.parse(responseEntity.getContent(), "UTF-8", url.toString());
+				Elements links = d.select("a[href]");
+				logger.trace("Found " + links.size() + " on " + url);
+				for (Element link : links) {
+					referred.add(link.attr("href"));
+				}
+			} else {
+				logger.trace("Not following links in content type " + contentType);
+			}
+		} else if (response.getFirstHeader("Location") != null) {
+			String location = response.getFirstHeader("Location").getValue();
+			if (referred.add(location)) {
+				logger.trace("Following redirect (" + statusCode + ") [" + url + " => " + location + "]");
+			}
+		} else {
+			logger.debug("Skipping {}, content-type: {}", url, contentType);
+		}
+		return referred;
+
+	}
+}