lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-25 23:25:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

for ARI-3712, add extracted links relative to both via and base, and annotate with "extractorSWFRelToVia", "extractorSWFRelToBase", or "extractorSWFRelToBoth" if resulting link is the same whether relative to base or via

Browse Source
This commit is contained in:
Noah Levitt
2014-05-15 13:31:29 -07:00
parent 519717fd7b
commit aff5b9ac4e
2 changed files with 64 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java
+24 -3
View File
@@ -334,17 +334,38 @@ public class ExtractorSWF extends ContentExtractor {
}
} else {
int max = ext.getExtractorParameters().getMaxOutlinks();
Link.addRelativeToVia(curi, max, url, LinkContext.EMBED_MISC,
Hop.EMBED);
Link relToVia = Link.addRelativeToVia(curi, max, url,
LinkContext.EMBED_MISC, Hop.EMBED);
Link relToBase = Link.addRelativeToBase(curi, max, url,
LinkContext.EMBED_MISC, Hop.EMBED);
addAnnotations(relToVia, relToBase);
linkCount++;
}
}
protected void addAnnotations(Link relToVia, Link relToBase) {
if (relToVia != null && relToBase != null
&& relToVia.getDestination().equals(relToBase.getDestination())) {
relToVia.getAnnotations().add("extractorSWFRelToBoth");
relToBase.getAnnotations().add("extractorSWFRelToBoth");
} else {
if (relToVia != null) {
relToVia.getAnnotations().add("extractorSWFRelToVia");
}
if (relToBase != null) {
relToBase.getAnnotations().add("extractorSWFRelToBase");
}
}
}
public void considerStringAsUri(String str) throws IOException {
if (UriUtils.isVeryLikelyUri(str)) {
int max = ext.getExtractorParameters().getMaxOutlinks();
Link.addRelativeToVia(curi, max, str,
Link relToVia = Link.addRelativeToVia(curi, max, str,
LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
Link relToBase = Link.addRelativeToBase(curi, max, str,
LinkContext.SPECULATIVE_MISC, Hop.SPECULATIVE);
addAnnotations(relToVia, relToBase);
linkCount++;
}
}
modules/src/test/java/org/archive/modules/extractor/ExtractorSWFTest.java
+40
View File
@@ -177,4 +177,44 @@ public class ExtractorSWFTest extends ContentExtractorTestBase {
+ "\" from " + url, foundIt);
}
}
public void xestAri3712() throws MalformedURLException, IOException {
String url = "https://wayback.archive-it.org/3771/20131119163257/http://nyumedecs.kk5.org/_app/28727/en/resources/container.swf";
CrawlURI curi = setupURI(url);
curi.setVia(UURIFactory.getInstance("http://nyumedecs.kk5.org/"));
long startTime = System.currentTimeMillis();
this.extractor.extract(curi);
long elapsed = System.currentTimeMillis() - startTime;
logger.info(this.extractor.getClass().getSimpleName() + " took "
+ elapsed + "ms to process " + url);
HashMap<CharSequence, String> expected = new HashMap<CharSequence, String>();
expected.put("http://nyumedecs.kk5.org/sm4/portal", "extractorSWFRelToVia");
expected.put("https://wayback.archive-it.org/3771/20131119163257/http://nyumedecs.kk5.org/_app/28727/en/resources/sm4/portal", "extractorSWFRelToBase");
expected.put("http://nyumedecs.kk5.org/", "extractorSWFRelToVia");
expected.put("https://wayback.archive-it.org/", "extractorSWFRelToBase");
expected.put("http://nyumedecs.kk5.org/loadingBarEdit.swf", "extractorSWFRelToVia");
expected.put("https://wayback.archive-it.org/3771/20131119163257/http://nyumedecs.kk5.org/_app/28727/en/resources/loadingBarEdit.swf", "extractorSWFRelToBase");
expected.put("http://nyumedecs.kk5.org/containermain.swf", "extractorSWFRelToVia");
expected.put("https://wayback.archive-it.org/3771/20131119163257/http://nyumedecs.kk5.org/_app/28727/en/resources/containermain.swf", "extractorSWFRelToBase");
for (Link link: curi.getOutLinks()) {
System.out.println(link + " " + link.getData());
assertEquals(1, link.getAnnotations().size());
String dest = link.getDestination().toString();
assertTrue(expected.containsKey(dest));
// remove the entry, so at the end the map should be empty, confirming that we found all the expected links
String expectedAnnotation = expected.remove(dest);
System.out.println("expectedAnnotation=" + expectedAnnotation);
System.out.println("link.getAnnotations()=" + link.getAnnotations());
String annotation = link.getAnnotations().toArray(new String[0])[0];
System.out.println("annotation=" + annotation);
assertEquals(expectedAnnotation, annotation);
}
}
}