lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 06:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

[HER-1480] URIs logged (as -6 or -7) that should never be scheduled

Browse Source 
* LaxURI.java
    increase tolerance with regard to path-segments
* UURIFactory.java
    allow digits in scheme regex
    discard old supported-schemes code
* UURIFactoryTest.java
    tests for roundtrip serialization of problematic hostlike-scheme and bars-in-path
* CrawlerLoggerModule.java
    discard special-casing of URI scheme errors
* SchemeNotInSetDecideRule.java
    rule to apply to URIs of unknown schemes; by default REJECTs those not usually handled by Heritrix
* **/profile-crawler-beans.cxml
    add REJECT SchemeNotInSetDecideRule
This commit is contained in:
gojomo
2010-09-03 00:24:13 +00:00
parent 0347269785
commit bc698c09fc
7 changed files with 118 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/net/LaxURI.java
+10
View File
@@ -53,6 +53,13 @@ public class LaxURI extends URI {
lax_abs_path.set('|'); // tests indicate Firefox (1.0.6) doesn't escape.
}
protected static final BitSet lax_rel_path = new BitSet(256);
// Static initializer for rel_path
static {
lax_rel_path.or(lax_rel_segment);
lax_rel_path.or(lax_abs_path);
}
protected static final BitSet lax_query = new BitSet(256);
static {
lax_query.or(query);
@@ -154,6 +161,9 @@ public class LaxURI extends URI {
if (generous == query) {
return lax_query;
}
if (generous == rel_path) {
return lax_rel_path;
}
// otherwise, leave as is
return generous;
}
commons/src/main/java/org/archive/net/UURIFactory.java
+3 -58
View File
@@ -23,7 +23,6 @@ import gnu.inet.encoding.IDNAException;
import it.unimi.dsi.mg4j.util.MutableString;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import java.util.BitSet;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -124,7 +123,7 @@ public class UURIFactory extends URI {
* (3) scheme is limited to legal scheme characters
*/
final public static Pattern RFC2396REGEX = Pattern.compile(
"^(([a-zA-Z][a-zA-Z\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?");
"^(([a-zA-Z][a-zA-Z0-9\\+\\-\\.]*):)?((//([^/?#]*))?([^?#]*)(\\?([^#]*))?)?(#(.*))?");
// 12 34 5 6 7 8 9 A
// 2 1 54 6 87 3 A9 // 1: scheme
// 2: scheme:
@@ -219,36 +218,11 @@ public class UURIFactory extends URI {
*/
final static Pattern MULTIPLE_SLASHES = Pattern.compile("//+");
/**
* System property key for list of supported schemes.
*/
private static final String SCHEMES_KEY = ".schemes";
/**
* System property key for list of purposefully-ignored schemes.
*/
public static final String IGNORED_SCHEMES_KEY = ".ignored-schemes";
private String[] schemes = null;
private String[] ignoredSchemes = null;
public static final int IGNORED_SCHEME = 9999999;
/**
* Protected constructor.
*/
private UURIFactory() {
super();
String s = System.getProperty(this.getClass().getName() + SCHEMES_KEY);
if (s != null && s.length() > 0) {
schemes = s.split("[, ]+");
Arrays.sort(schemes);
}
String ignored = System.getProperty(this.getClass().getName() + IGNORED_SCHEMES_KEY);
if (ignored != null && ignored.length() > 0) {
ignoredSchemes = ignored.split("[, ]+");
Arrays.sort(ignoredSchemes);
}
}
/**
@@ -279,24 +253,9 @@ public class UURIFactory extends URI {
*/
public static UURI getInstance(UURI base, String relative)
throws URIException {
// return base.resolve(relative);
return UURIFactory.factory.create(base, relative);
}
/**
* Test of whether passed String has an allowed URI scheme.
* First tests if likely scheme suffix. If so, we then test if its one of
* the supported schemes.
* @param possibleUrl URL string to examine.
* @return True if passed string looks like it could be an URL.
*/
public static boolean hasSupportedScheme(String possibleUrl) {
boolean hasScheme = UURI.hasScheme(possibleUrl);
if (!hasScheme || UURIFactory.factory.schemes == null) {
return hasScheme;
}
String tmpStr = possibleUrl.substring(0, possibleUrl.indexOf(':'));
return Arrays.binarySearch(UURIFactory.factory.schemes, tmpStr) >= 0;
}
/**
* @param uri URI as string.
@@ -422,7 +381,7 @@ public class UURIFactory extends URI {
uri = escapeWhitespace(uri);
// For further processing, get uri elements. See the RFC2396REGEX
// comment above for explaination of group indices used in the below.
// comment above for explanation of group indices used in the below.
matcher = RFC2396REGEX.matcher(uri);
if (!matcher.matches()) {
throw new URIException("Failed parse of " + uri);
@@ -434,20 +393,6 @@ public class UURIFactory extends URI {
String uriQuery = checkUriElement(matcher.group(8));
// UNUSED String uriFragment = checkUriElement(matcher.group(10));
// If a scheme, is it a supported scheme?
if (uriScheme != null && uriScheme.length() > 0 &&
this.schemes != null) {
if (!(Arrays.binarySearch(schemes,uriScheme)>=0)) {
// unsupported; see if silently ignored
if((Arrays.binarySearch(ignoredSchemes,uriScheme)>=0)) {
throw new URIException(
IGNORED_SCHEME, "Ignored scheme: " + uriScheme);
} else {
throw new URIException("Unsupported scheme: " + uriScheme);
}
}
}
// Test if relative URI. If so, need a base to resolve against.
if (uriScheme == null || uriScheme.length() <= 0) {
if (base == null) {
commons/src/test/java/org/archive/net/UURIFactoryTest.java
+31
View File
@@ -1123,6 +1123,24 @@ public class UURIFactoryTest extends TestCase {
// uuri2 = UURIFactory.getInstance(uuri.toCustomString());
// assertEquals("Not equal", uuri.toString(), uuri2.toString());
}
/**
* A UURI's string representation should be same after a
* toCustomString-getInstance roundtrip.
*
* @throws URIException
*/
public final void testHostnamePortRoundtrip() throws URIException {
UURI base = UURIFactory.
getInstance("http://www.example.com/path?query#anchor");
UURI test = UURIFactory.getInstance(base,"boom1.hostname.com:9999");
System.out.println("scheme:"+test.getScheme());
System.out.println(test.toCustomString());
UURI roundtrip = UURIFactory.getInstance(test.toCustomString());
assertEquals("Not equal", test.toString(), roundtrip.toString());
}
/**
* Test bad port throws URIException not NumberFormatException
*/
@@ -1135,4 +1153,17 @@ public class UURIFactoryTest extends TestCase {
// expected
}
}
/**
* Bars ('|') in path-segments aren't encoded by FF, preferred by some
* RESTful-URI-ideas guides, so should work without error.
*
* @throws URIException
*/
public void testBarsInRelativePath() throws URIException {
UURI base = UURIFactory.getInstance("http://www.example.com");
String relative = "foo/bar|baz|yorple";
base.resolve(relative);
UURIFactory.getInstance(base,relative);
}
}
dist/src/main/conf/jobs/profile-defaults/profile-crawler-beans.cxml
Vendored
+3
View File
@@ -156,6 +156,9 @@ http://example.example/example
<!-- ...but always ACCEPT those marked as prerequisitee for another URI... -->
<bean class="org.archive.modules.deciderules.PrerequisiteAcceptDecideRule">
</bean>
<!-- ...but always REJECT those with unsupported URI schemes -->
<bean class="org.archive.modules.deciderules.SchemeNotInSetDecideRule">
</bean>
</list>
</property>
</bean>
engine/src/main/java/org/archive/crawler/reporting/CrawlerLoggerModule.java
-5
View File
@@ -43,7 +43,6 @@ import org.archive.crawler.util.Logs;
import org.archive.io.GenerationFileHandler;
import org.archive.modules.extractor.UriErrorLoggerModule;
import org.archive.net.UURI;
import org.archive.net.UURIFactory;
import org.archive.spring.ConfigPath;
import org.archive.util.ArchiveUtils;
import org.springframework.beans.factory.InitializingBean;
@@ -418,10 +417,6 @@ public class CrawlerLoggerModule
* @param l String which could not be interpreted as URI without exception
*/
public void logUriError(URIException e, UURI u, CharSequence l) {
if (e.getReasonCode() == UURIFactory.IGNORED_SCHEME) {
// don't log those that are intentionally ignored
return;
}
Object[] array = {u, l};
uriErrors.log(Level.INFO, e.getMessage(), array);
}
engine/src/main/resources/org/archive/crawler/restlet/profile-crawler-beans.cxml
+3
View File
@@ -156,6 +156,9 @@ http://example.example/example
<!-- ...but always ACCEPT those marked as prerequisitee for another URI... -->
<bean class="org.archive.modules.deciderules.PrerequisiteAcceptDecideRule">
</bean>
<!-- ...but always REJECT those with unsupported URI schemes -->
<bean class="org.archive.modules.deciderules.SchemeNotInSetDecideRule">
</bean>
</list>
</property>
</bean>
modules/src/main/java/org/archive/modules/deciderules/SchemeNotInSetDecideRule.java
+68
View File
@@ -0,0 +1,68 @@
/*
* This file is part of the Heritrix web crawler (crawler.archive.org).
*
* Licensed to the Internet Archive (IA) by one or more individual
* contributors.
*
* The IA licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.modules.deciderules;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import org.archive.modules.CrawlURI;
/**
* Rule applies the configured decision (default REJECT) for any URI which
* has a URI-scheme NOT contained in the configured Set.
*
* @contributor gojomo
*/
public class SchemeNotInSetDecideRule extends PredicatedDecideRule {
private static final long serialVersionUID = 3L;
{
setDecision(DecideResult.REJECT);
}
/**
* Usual constructor.
* @param name Name of this DecideRule.
*/
public SchemeNotInSetDecideRule() {
}
/** set of schemes to test URI scheme */
protected Set<String> schemes = new HashSet<String>();
{
// default set are those schemes Heritrix supports in usual configuration
schemes.addAll(Arrays.asList(new String[] {"http","https","ftp","dns"}));
}
public Set<String> getSchemes() {
return schemes;
}
public void setSchemes(Set<String> schemes) {
this.schemes = schemes;
}
/**
* Evaluate whether given object is over the threshold number of
* hops.
*/
@Override
protected boolean evaluate(CrawlURI uri) {
return !schemes.contains(uri.getUURI().getScheme());
}
}