lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-27 16:15:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #231 from ukwa/synchronized-enqueuecounts

Browse Source 
Add synchronized statements for internetarchive/heritrix3#221.
This commit is contained in:
Andy Jackson
2019-02-07 13:49:50 +00:00
committed by GitHub
parent cb96a44bdc 3b435cb0a9
commit 18d3bb23aa
1 changed files with 32 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java
+32 -19
View File
@@ -18,11 +18,11 @@
*/
package org.archive.crawler.frontier.precedence;
import java.io.PrintWriter;
import org.archive.crawler.frontier.WorkQueue;
import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStats.Stage;
import java.io.PrintWriter;
import org.archive.crawler.frontier.WorkQueue;
import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStats.Stage;
import org.archive.util.Histotable;
/**
@@ -65,14 +65,18 @@ public class HighestUriQueuePrecedencePolicy extends BaseQueuePrecedencePolicy {
public void tally(CrawlURI curi, Stage stage) {
switch(stage) {
case SCHEDULED:
// enqueued
enqueuedCounts.tally(curi.getPrecedence());
// enqueued
synchronized (enqueuedCounts) {
enqueuedCounts.tally(curi.getPrecedence());
}
break;
case SUCCEEDED:
case DISREGARDED:
case FAILED:
// dequeued
enqueuedCounts.tally(curi.getPrecedence(),-1);
synchronized (enqueuedCounts) {
enqueuedCounts.tally(curi.getPrecedence(), -1);
}
break;
case RETRIED:
// do nothing, already tallied
@@ -85,9 +89,13 @@ public class HighestUriQueuePrecedencePolicy extends BaseQueuePrecedencePolicy {
*/
@Override
public int getPrecedence() {
// base plus highest URI still in queue
Integer delta = (enqueuedCounts.size() > 0) ? enqueuedCounts.firstKey() : 0;
return super.getPrecedence() + delta;
// base plus highest URI still in queue
synchronized (enqueuedCounts) {
Integer delta = (enqueuedCounts.size() > 0)
? enqueuedCounts.firstKey()
: 0;
return super.getPrecedence() + delta;
}
}
/*
@@ -98,20 +106,25 @@ public class HighestUriQueuePrecedencePolicy extends BaseQueuePrecedencePolicy {
StringBuilder sb = new StringBuilder();
sb.append(super.shortReportLegend());
sb.append(":");
for(Integer p : enqueuedCounts.keySet()) {
sb.append(" p");
sb.append(p);
synchronized (enqueuedCounts) {
for (Integer p : enqueuedCounts.keySet()) {
sb.append(" p");
sb.append(p);
}
}
return sb.toString();
}
@Override
public void shortReportLineTo(PrintWriter writer) {
boolean betwixt = false;
for(Long count : enqueuedCounts.values()) {
if(betwixt) writer.print(" ");
writer.print(count);
betwixt = true;
boolean betwixt = false;
synchronized (enqueuedCounts) {
for (Long count : enqueuedCounts.values()) {
if (betwixt)
writer.print(" ");
writer.print(count);
betwixt = true;
}
}
}