Log warning when URI precedence exceeds maximum 127

BdbMultipleWorkQueues.calculateInsertKey() silently clips precedence
values to 127 for queue ordering. This can cause unexpected behavior
for custom cost assignment policies that return values above 127.

Log a warning when clipping occurs and fix the misleading comment in
CrawlURI.holderCost that said "should not exceed 255" (the actual
limit is 127, not 255).

Fixes #502
This commit is contained in:
Simon
2026-03-31 00:37:36 -07:00
parent bc82cf5ab8
commit 47632eb4c4
2 changed files with 8 additions and 2 deletions
@@ -434,7 +434,13 @@ public class BdbMultipleWorkQueues {
keyData[len] = 0;
long ordinalPlus = curi.getOrdinal() & 0x0000FFFFFFFFFFFFL;
ordinalPlus = ((long) curi.getSchedulingDirective() << 56) | ordinalPlus;
long precedence = Math.min(curi.getPrecedence(), 127);
long rawPrecedence = curi.getPrecedence();
if (rawPrecedence > 127) {
LOGGER.warning("URI precedence " + rawPrecedence
+ " exceeds maximum 127 and will be clipped"
+ " for queue ordering: " + curi);
}
long precedence = Math.min(rawPrecedence, 127);
ordinalPlus = (((precedence) & 0xFFL) << 48) | ordinalPlus;
ArchiveUtils.longIntoByteArray(ordinalPlus, keyData, len + 1);
return new DatabaseEntry(keyData);
@@ -997,7 +997,7 @@ implements Reporter, Serializable, OverlayContext, Comparable<CrawlURI> {
}
/** spot for an integer cost to be placed by external facility (frontier).
* cost is truncated to 8 bits at times, so should not exceed 255 */
* cost is truncated to 7 bits at times, so should not exceed 127 */
protected int holderCost = UNCALCULATED;
/**
* Return the 'holderCost' for convenience of external facility (frontier)