diff --git a/commons/src/main/java/org/archive/net/LaxURI.java b/commons/src/main/java/org/archive/net/LaxURI.java
index 4e71aebc..fac6d21b 100644
--- a/commons/src/main/java/org/archive/net/LaxURI.java
+++ b/commons/src/main/java/org/archive/net/LaxURI.java
@@ -53,6 +53,13 @@ public class LaxURI extends URI {
lax_abs_path.set('|'); // tests indicate Firefox (1.0.6) doesn't escape.
}
+ protected static final BitSet lax_rel_path = new BitSet(256);
+ // Static initializer for rel_path
+ static {
+ lax_rel_path.or(lax_rel_segment);
+ lax_rel_path.or(lax_abs_path);
+ }
+
protected static final BitSet lax_query = new BitSet(256);
static {
lax_query.or(query);
@@ -154,6 +161,9 @@ public class LaxURI extends URI {
if (generous == query) {
return lax_query;
}
+ if (generous == rel_path) {
+ return lax_rel_path;
+ }
// otherwise, leave as is
return generous;
}
diff --git a/commons/src/main/java/org/archive/net/UURIFactory.java b/commons/src/main/java/org/archive/net/UURIFactory.java
index 7aa9fa18..1c29b036 100644
--- a/commons/src/main/java/org/archive/net/UURIFactory.java
+++ b/commons/src/main/java/org/archive/net/UURIFactory.java
@@ -23,7 +23,6 @@ import gnu.inet.encoding.IDNAException;
import it.unimi.dsi.mg4j.util.MutableString;
import java.io.UnsupportedEncodingException;
-import java.util.Arrays;
import java.util.BitSet;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -124,7 +123,7 @@ public class UURIFactory extends URI {
* (3) scheme is limited to legal scheme characters
*/
final public static Pattern RFC2396REGEX = Pattern.compile(
- "^(([a-zA-Z][a-zA-Z\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?");
+ "^(([a-zA-Z][a-zA-Z0-9\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?");
// 12 34 5 6 7 8 9 A
// 2 1 54 6 87 3 A9 // 1: scheme
// 2: scheme:
@@ -219,36 +218,11 @@ public class UURIFactory extends URI {
*/
final static Pattern MULTIPLE_SLASHES = Pattern.compile("//+");
- /**
- * System property key for list of supported schemes.
- */
- private static final String SCHEMES_KEY = ".schemes";
-
- /**
- * System property key for list of purposefully-ignored schemes.
- */
- public static final String IGNORED_SCHEMES_KEY = ".ignored-schemes";
-
- private String[] schemes = null;
- private String[] ignoredSchemes = null;
-
- public static final int IGNORED_SCHEME = 9999999;
-
/**
* Protected constructor.
*/
private UURIFactory() {
super();
- String s = System.getProperty(this.getClass().getName() + SCHEMES_KEY);
- if (s != null && s.length() > 0) {
- schemes = s.split("[, ]+");
- Arrays.sort(schemes);
- }
- String ignored = System.getProperty(this.getClass().getName() + IGNORED_SCHEMES_KEY);
- if (ignored != null && ignored.length() > 0) {
- ignoredSchemes = ignored.split("[, ]+");
- Arrays.sort(ignoredSchemes);
- }
}
/**
@@ -279,24 +253,9 @@ public class UURIFactory extends URI {
*/
public static UURI getInstance(UURI base, String relative)
throws URIException {
+// return base.resolve(relative);
return UURIFactory.factory.create(base, relative);
}
-
- /**
- * Test of whether passed String has an allowed URI scheme.
- * First tests if likely scheme suffix. If so, we then test if its one of
- * the supported schemes.
- * @param possibleUrl URL string to examine.
- * @return True if passed string looks like it could be an URL.
- */
- public static boolean hasSupportedScheme(String possibleUrl) {
- boolean hasScheme = UURI.hasScheme(possibleUrl);
- if (!hasScheme || UURIFactory.factory.schemes == null) {
- return hasScheme;
- }
- String tmpStr = possibleUrl.substring(0, possibleUrl.indexOf(':'));
- return Arrays.binarySearch(UURIFactory.factory.schemes, tmpStr) >= 0;
- }
/**
* @param uri URI as string.
@@ -422,7 +381,7 @@ public class UURIFactory extends URI {
uri = escapeWhitespace(uri);
// For further processing, get uri elements. See the RFC2396REGEX
- // comment above for explaination of group indices used in the below.
+ // comment above for explanation of group indices used in the below.
matcher = RFC2396REGEX.matcher(uri);
if (!matcher.matches()) {
throw new URIException("Failed parse of " + uri);
@@ -434,20 +393,6 @@ public class UURIFactory extends URI {
String uriQuery = checkUriElement(matcher.group(8));
// UNUSED String uriFragment = checkUriElement(matcher.group(10));
- // If a scheme, is it a supported scheme?
- if (uriScheme != null && uriScheme.length() > 0 &&
- this.schemes != null) {
- if (!(Arrays.binarySearch(schemes,uriScheme)>=0)) {
- // unsupported; see if silently ignored
- if((Arrays.binarySearch(ignoredSchemes,uriScheme)>=0)) {
- throw new URIException(
- IGNORED_SCHEME, "Ignored scheme: " + uriScheme);
- } else {
- throw new URIException("Unsupported scheme: " + uriScheme);
- }
- }
- }
-
// Test if relative URI. If so, need a base to resolve against.
if (uriScheme == null || uriScheme.length() <= 0) {
if (base == null) {
diff --git a/commons/src/test/java/org/archive/net/UURIFactoryTest.java b/commons/src/test/java/org/archive/net/UURIFactoryTest.java
index 9fe3b55f..c6b90d86 100644
--- a/commons/src/test/java/org/archive/net/UURIFactoryTest.java
+++ b/commons/src/test/java/org/archive/net/UURIFactoryTest.java
@@ -1123,6 +1123,24 @@ public class UURIFactoryTest extends TestCase {
// uuri2 = UURIFactory.getInstance(uuri.toCustomString());
// assertEquals("Not equal", uuri.toString(), uuri2.toString());
}
+
+ /**
+ * A UURI's string representation should be same after a
+ * toCustomString-getInstance roundtrip.
+ *
+ * @throws URIException
+ */
+ public final void testHostnamePortRoundtrip() throws URIException {
+ UURI base = UURIFactory.
+ getInstance("http://www.example.com/path?query#anchor");
+ UURI test = UURIFactory.getInstance(base,"boom1.hostname.com:9999");
+ System.out.println("scheme:"+test.getScheme());
+ System.out.println(test.toCustomString());
+ UURI roundtrip = UURIFactory.getInstance(test.toCustomString());
+ assertEquals("Not equal", test.toString(), roundtrip.toString());
+ }
+
+
/**
* Test bad port throws URIException not NumberFormatException
*/
@@ -1135,4 +1153,17 @@ public class UURIFactoryTest extends TestCase {
// expected
}
}
+
+ /**
+ * Bars ('|') in path-segments aren't encoded by FF, preferred by some
+ * RESTful-URI-ideas guides, so should work without error.
+ *
+ * @throws URIException
+ */
+ public void testBarsInRelativePath() throws URIException {
+ UURI base = UURIFactory.getInstance("http://www.example.com");
+ String relative = "foo/bar|baz|yorple";
+ base.resolve(relative);
+ UURIFactory.getInstance(base,relative);
+ }
}
diff --git a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml
index 950acade..1739330e 100644
--- a/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml
+++ b/dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml
@@ -156,6 +156,9 @@ http://example.example/example
+
+
+
diff --git a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java
index 00bea67e..29336746 100644
--- a/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java
+++ b/engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java
@@ -43,7 +43,6 @@ import org.archive.crawler.util.Logs;
import org.archive.io.GenerationFileHandler;
import org.archive.modules.extractor.UriErrorLoggerModule;
import org.archive.net.UURI;
-import org.archive.net.UURIFactory;
import org.archive.spring.ConfigPath;
import org.archive.util.ArchiveUtils;
import org.springframework.beans.factory.InitializingBean;
@@ -418,10 +417,6 @@ public class CrawlerLoggerModule
* @param l String which could not be interpreted as URI without exception
*/
public void logUriError(URIException e, UURI u, CharSequence l) {
- if (e.getReasonCode() == UURIFactory.IGNORED_SCHEME) {
- // don't log those that are intentionally ignored
- return;
- }
Object[] array = {u, l};
uriErrors.log(Level.INFO, e.getMessage(), array);
}
diff --git a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
index 319a150d..ec4f7189 100644
--- a/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
+++ b/engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
@@ -156,6 +156,9 @@ http://example.example/example
+
+
+
diff --git a/modules/src/main/java/org/archive/modules/deciderules/SchemeNotInSetDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/SchemeNotInSetDecideRule.java
new file mode 100644
index 00000000..9c076625
--- /dev/null
+++ b/modules/src/main/java/org/archive/modules/deciderules/SchemeNotInSetDecideRule.java
@@ -0,0 +1,68 @@
+/*
+ * This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ * Licensed to the Internet Archive (IA) by one or more individual
+ * contributors.
+ *
+ * The IA licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.archive.modules.deciderules;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.archive.modules.CrawlURI;
+
+/**
+ * Rule applies the configured decision (default REJECT) for any URI which
+ * has a URI-scheme NOT contained in the configured Set.
+ *
+ * @contributor gojomo
+ */
+public class SchemeNotInSetDecideRule extends PredicatedDecideRule {
+ private static final long serialVersionUID = 3L;
+
+ {
+ setDecision(DecideResult.REJECT);
+ }
+
+ /**
+ * Usual constructor.
+ * @param name Name of this DecideRule.
+ */
+ public SchemeNotInSetDecideRule() {
+ }
+
+ /** set of schemes to test URI scheme */
+ protected Set schemes = new HashSet();
+ {
+ // default set are those schemes Heritrix supports in usual configuration
+ schemes.addAll(Arrays.asList(new String[] {"http","https","ftp","dns"}));
+ }
+ public Set getSchemes() {
+ return schemes;
+ }
+ public void setSchemes(Set schemes) {
+ this.schemes = schemes;
+ }
+
+ /**
+ * Evaluate whether given object is over the threshold number of
+ * hops.
+ */
+ @Override
+ protected boolean evaluate(CrawlURI uri) {
+ return !schemes.contains(uri.getUURI().getScheme());
+ }
+}