lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-08-28 09:26:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: fix line endings

Browse Source
This commit is contained in:
Adam Miller
2025-04-22 17:33:32 -07:00
parent e7fe566919
commit cb98bfc522
1 changed files with 84 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine/src/main/java/org/archive/crawler/frontier/precedence/HighestUriQueuePrecedencePolicy.java
+84 -84
View File
@@ -16,108 +16,108 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.archive.crawler.frontier.precedence;
package org.archive.crawler.frontier.precedence;
import java.io.PrintWriter;
import org.archive.bdb.AutoKryo;
import org.archive.crawler.frontier.WorkQueue;
import org.archive.modules.CrawlURI;
import org.archive.modules.fetcher.FetchStats.Stage;
import org.archive.util.Histotable;
/**
* QueuePrecedencePolicy that sets a uri-queue's precedence to that of the
* highest URI currently enqueued within itself, added to the configured
* base-precedence.
*
* It does this by maintaining a count of the included URIs at each
* URI-precedence, updated on each URI add or remove.
*/
public class HighestUriQueuePrecedencePolicy extends BaseQueuePrecedencePolicy {
private static final long serialVersionUID = -8652293180921419601L;
/* (non-Javadoc)
* @see org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy#installProvider(org.archive.crawler.frontier.WorkQueue)
*/
@Override
protected void installProvider(WorkQueue wq) {
// TODO:SPRINGY ensure proper override context installed for getBasePrecedence() below
HighestUriPrecedenceProvider provider = new HighestUriPrecedenceProvider(getBasePrecedence());
wq.setPrecedenceProvider(provider);
}
/**
* Helper provider for maintaining the tracked distribution of included
* URIs and calculating the queue precedence.
*/
public class HighestUriPrecedenceProvider extends SimplePrecedenceProvider {
private static final long serialVersionUID = 5545297542888582745L;
protected Histotable<Integer> enqueuedCounts = new Histotable<Integer>();
public HighestUriPrecedenceProvider(int base) {
super(base);
}
/* (non-Javadoc)
* @see org.archive.crawler.frontier.precedence.PrecedenceProvider#tally(org.archive.modules.CrawlURI, org.archive.modules.fetcher.FetchStats.Stage)
*/
@Override
public void tally(CrawlURI curi, Stage stage) {
switch(stage) {
case SCHEDULED:
import org.archive.util.Histotable;
/**
* QueuePrecedencePolicy that sets a uri-queue's precedence to that of the
* highest URI currently enqueued within itself, added to the configured
* base-precedence.
*
* It does this by maintaining a count of the included URIs at each
* URI-precedence, updated on each URI add or remove.
*/
public class HighestUriQueuePrecedencePolicy extends BaseQueuePrecedencePolicy {
private static final long serialVersionUID = -8652293180921419601L;
/* (non-Javadoc)
* @see org.archive.crawler.frontier.precedence.BaseQueuePrecedencePolicy#installProvider(org.archive.crawler.frontier.WorkQueue)
*/
@Override
protected void installProvider(WorkQueue wq) {
// TODO:SPRINGY ensure proper override context installed for getBasePrecedence() below
HighestUriPrecedenceProvider provider = new HighestUriPrecedenceProvider(getBasePrecedence());
wq.setPrecedenceProvider(provider);
}
/**
* Helper provider for maintaining the tracked distribution of included
* URIs and calculating the queue precedence.
*/
public class HighestUriPrecedenceProvider extends SimplePrecedenceProvider {
private static final long serialVersionUID = 5545297542888582745L;
protected Histotable<Integer> enqueuedCounts = new Histotable<Integer>();
public HighestUriPrecedenceProvider(int base) {
super(base);
}
/* (non-Javadoc)
* @see org.archive.crawler.frontier.precedence.PrecedenceProvider#tally(org.archive.modules.CrawlURI, org.archive.modules.fetcher.FetchStats.Stage)
*/
@Override
public void tally(CrawlURI curi, Stage stage) {
switch(stage) {
case SCHEDULED:
// enqueued
synchronized (enqueuedCounts) {
enqueuedCounts.tally(curi.getPrecedence());
}
break;
case SUCCEEDED:
case DISREGARDED:
case FAILED:
// dequeued
}
break;
case SUCCEEDED:
case DISREGARDED:
case FAILED:
// dequeued
synchronized (enqueuedCounts) {
enqueuedCounts.tally(curi.getPrecedence(), -1);
}
break;
case RETRIED:
// do nothing, already tallied
break;
}
}
/* (non-Javadoc)
* @see org.archive.crawler.frontier.precedence.SimplePrecedenceProvider#getPrecedence()
*/
@Override
public int getPrecedence() {
}
break;
case RETRIED:
// do nothing, already tallied
break;
}
}
/* (non-Javadoc)
* @see org.archive.crawler.frontier.precedence.SimplePrecedenceProvider#getPrecedence()
*/
@Override
public int getPrecedence() {
// base plus highest URI still in queue
synchronized (enqueuedCounts) {
Integer delta = (enqueuedCounts.size() > 0)
? enqueuedCounts.firstKey()
: 0;
return super.getPrecedence() + delta;
}
}
/*
* @see org.archive.crawler.frontier.precedence.PrecedenceProvider#shortReportLegend()()
*/
@Override
public String shortReportLegend() {
StringBuilder sb = new StringBuilder();
sb.append(super.shortReportLegend());
sb.append(":");
}
}
/*
* @see org.archive.crawler.frontier.precedence.PrecedenceProvider#shortReportLegend()()
*/
@Override
public String shortReportLegend() {
StringBuilder sb = new StringBuilder();
sb.append(super.shortReportLegend());
sb.append(":");
synchronized (enqueuedCounts) {
for (Integer p : enqueuedCounts.keySet()) {
sb.append(" p");
sb.append(p);
}
}
return sb.toString();
}
@Override
public void shortReportLineTo(PrintWriter writer) {
}
return sb.toString();
}
@Override
public void shortReportLineTo(PrintWriter writer) {
boolean betwixt = false;
synchronized (enqueuedCounts) {
for (Long count : enqueuedCounts.values()) {
@@ -126,13 +126,13 @@ public class HighestUriQueuePrecedencePolicy extends BaseQueuePrecedencePolicy {
writer.print(count);
betwixt = true;
}
}
}
}
}
// Kryo support
public static void autoregisterTo(AutoKryo kryo) {
kryo.register(HighestUriPrecedenceProvider.class);
kryo.autoregister(org.archive.util.Histotable.class);
}
}
}
}
}