lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-07-20 14:31:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix HER-2021 usedBaseForVia annotation repeated many times

Browse Source 
* CrawlURI.java
    getAnnotations() - use LinkedHashSet data structure for new annotations object to prevent duplicates
* Link.java
    addRelativeToVia() - only log use of base for via the first time, before the annotation has been added
This commit is contained in:
Noah Levitt
2012-09-04 17:49:01 -07:00
parent d0882a18e6
commit b704f4eeb7
2 changed files with 13 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/CrawlURI.java
+6 -5
View File
@@ -66,6 +66,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -651,12 +652,12 @@ implements Reporter, Serializable, OverlayContext {
*/
public Collection<String> getAnnotations() {
@SuppressWarnings("unchecked")
List<String> list = (List<String>)getData().get(A_ANNOTATIONS);
if (list == null) {
list = new ArrayList<String>();
getData().put(A_ANNOTATIONS, list);
Collection<String> annotations = (Collection<String>)getData().get(A_ANNOTATIONS);
if (annotations == null) {
annotations = new LinkedHashSet<String>();
getData().put(A_ANNOTATIONS, annotations);
}
return list;
return annotations;
}
/**
modules/src/main/java/org/archive/modules/extractor/Link.java
+7 -6
View File
@@ -133,19 +133,20 @@ public class Link implements Serializable, Comparable<Link> {
}
public static void addRelativeToVia(CrawlURI uri, int max,
String newUri, LinkContext context, Hop hop) throws URIException {
public static void addRelativeToVia(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI relTo = uri.getVia();
if(relTo==null) {
LOGGER.info("no via where expected; using base instead: "+uri);
uri.getAnnotations().add("usedBaseForVia");
if (relTo == null) {
if (!uri.getAnnotations().contains("usedBaseForVia")) {
LOGGER.info("no via where expected; using base instead: " + uri);
uri.getAnnotations().add("usedBaseForVia");
}
relTo = uri.getBaseURI();
}
UURI dest = UURIFactory.getInstance(relTo, newUri);
add2(uri, max, dest, context, hop);
}
public static void add(CrawlURI uri, int max, String newUri,
LinkContext context, Hop hop) throws URIException {
UURI dest = UURIFactory.getInstance(newUri);