lee/heritrix3
1
0 Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-11-09 15:36:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix for [HER-1291] accurate scoping not possible at log-recovery time; URIs may be recrawled against expectations

Browse Source 
* FrontierJournal.java
    refactor to always operate on CrawlURI instances (with hops-path and via info), and to write longer info on success/failure lines
This commit is contained in:
gojomo committed 2009-05-13 01:43:41 +00:00
1 parent a530af6e5c
commit ab6942a60d
1 file changed
+10 -22
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java
+10 -22
View File
@@ -88,10 +88,13 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
timestamp_interval = 10000;
}
public synchronized void added(CrawlURI curi) {
public void added(CrawlURI curi) {
writeLongUriLine(F_ADD, curi);
}
public synchronized void writeLongUriLine(String tag, CrawlURI curi) {
accumulatingBuffer.length(0);
this.accumulatingBuffer.append(F_ADD).
this.accumulatingBuffer.append(tag).
append(curi.toString()).
append(" ").
append(curi.getPathFromSeed()).
@@ -101,15 +104,7 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
}
public void finishedSuccess(CrawlURI curi) {
finishedSuccess(curi.toString());
}
public void finishedSuccess(UURI uuri) {
finishedSuccess(uuri.toString());
}
protected void finishedSuccess(String uuri) {
writeLine(F_SUCCESS, uuri);
writeLongUriLine(F_SUCCESS, curi);
}
public void emitted(CrawlURI curi) {
@@ -118,20 +113,12 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
}
public void finishedFailure(CrawlURI curi) {
finishedFailure(curi.toString());
writeLongUriLine(F_FAILURE,curi);
}
public void finishedDisregard(CrawlURI curi) {
writeLine(F_DISREGARD, curi.toString());
}
public void finishedFailure(UURI uuri) {
finishedFailure(uuri.toString());
}
public void finishedFailure(String u) {
writeLine(F_FAILURE, u);
}
public void rescheduled(CrawlURI curi) {
writeLine(F_RESCHEDULE, curi.toString());
@@ -234,7 +221,8 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
}
frontier.considerIncluded(u);
if (newJournal != null) {
newJournal.writeLine(lineType, u.toString());
// write same line as read
newJournal.writeLine(read);
}
} catch (URIException e) {
e.printStackTrace();