mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-08-28 09:26:54 +00:00
Log warning when URI precedence exceeds maximum 127
BdbMultipleWorkQueues.calculateInsertKey() silently clips precedence values to 127 for queue ordering. This can cause unexpected behavior for custom cost assignment policies that return values above 127. Log a warning when clipping occurs and fix the misleading comment in CrawlURI.holderCost that said "should not exceed 255" (the actual limit is 127, not 255). Fixes #502
This commit is contained in:
@@ -434,7 +434,13 @@ public class BdbMultipleWorkQueues {
|
||||
keyData[len] = 0;
|
||||
long ordinalPlus = curi.getOrdinal() & 0x0000FFFFFFFFFFFFL;
|
||||
ordinalPlus = ((long) curi.getSchedulingDirective() << 56) | ordinalPlus;
|
||||
long precedence = Math.min(curi.getPrecedence(), 127);
|
||||
long rawPrecedence = curi.getPrecedence();
|
||||
if (rawPrecedence > 127) {
|
||||
LOGGER.warning("URI precedence " + rawPrecedence
|
||||
+ " exceeds maximum 127 and will be clipped"
|
||||
+ " for queue ordering: " + curi);
|
||||
}
|
||||
long precedence = Math.min(rawPrecedence, 127);
|
||||
ordinalPlus = (((precedence) & 0xFFL) << 48) | ordinalPlus;
|
||||
ArchiveUtils.longIntoByteArray(ordinalPlus, keyData, len + 1);
|
||||
return new DatabaseEntry(keyData);
|
||||
|
||||
@@ -997,7 +997,7 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
|
||||
}
|
||||
|
||||
/** spot for an integer cost to be placed by external facility (frontier).
|
||||
* cost is truncated to 8 bits at times, so should not exceed 255 */
|
||||
* cost is truncated to 7 bits at times, so should not exceed 127 */
|
||||
protected int holderCost = UNCALCULATED;
|
||||
/**
|
||||
* Return the 'holderCost' for convenience of external facility (frontier)
|
||||
|
||||
Reference in New Issue
Block a user