From 914756f155d7d4739f2eb9f1f13e69d7f855fc4b Mon Sep 17 00:00:00 2001
From: Alex Osborne <aosborne@nla.gov.au>
Date: Fri, 22 Apr 2022 16:43:37 +0900
Subject: [PATCH] ExtractorHTML: Add a main() method to run the extractor
 standalone

This makes troubleshooting link extraction problems much easier.
---
 .../modules/extractor/ExtractorHTML.java      | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
index 98d942bd..3fde6acd 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
@@ -20,9 +20,12 @@
 package org.archive.modules.extractor;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.net.URL;
 import java.net.URLDecoder;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Locale;
@@ -33,6 +36,7 @@ import java.util.regex.Pattern;
 
 import com.google.common.base.Ascii;
 import org.apache.commons.httpclient.URIException;
+import org.apache.commons.io.IOUtils;
 import org.archive.io.ReplayCharSequence;
 import org.archive.modules.CoreAttributeConstants;
 import org.archive.modules.CrawlMetadata;
@@ -1092,5 +1096,29 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
     public static CharSequence elementContext(CharSequence element, CharSequence attribute) {
         return attribute == null? "": (element + "/@" + attribute).toLowerCase(Locale.ROOT);
     }
+
+    public static void main(String[] args) throws Exception {
+        if (args.length == 0 || args[0].equals("-h") || args[0].equals("--help")) {
+            System.err.println("Usage: ExtractorHTML URL");
+            System.err.println("Extracts and prints links from the given URL");
+            System.exit(1);
+        }
+
+        String url = args[0];
+        CrawlURI curi = new CrawlURI(UURIFactory.getInstance(url));
+
+        ExtractorHTML extractor = new ExtractorHTML();
+        extractor.setExtractorJS(new ExtractorJS());
+        extractor.afterPropertiesSet();
+
+        String content;
+        try (InputStream stream = new URL(url).openStream()) {
+            content = IOUtils.toString(stream, StandardCharsets.ISO_8859_1);
+        }
+        extractor.extract(curi, content);
+        for (CrawlURI link : curi.getOutLinks()) {
+            System.out.println(link.getURI() + " " + link.getLastHop() + " " + link.getViaContext());
+        }
+    }
 }