From 47632eb4c4e80c0b28122d9a62175c4ecefb9dca Mon Sep 17 00:00:00 2001 From: Simon Date: Tue, 31 Mar 2026 00:37:36 -0700 Subject: [PATCH] Log warning when URI precedence exceeds maximum 127 BdbMultipleWorkQueues.calculateInsertKey() silently clips precedence values to 127 for queue ordering. This can cause unexpected behavior for custom cost assignment policies that return values above 127. Log a warning when clipping occurs and fix the misleading comment in CrawlURI.holderCost that said "should not exceed 255" (the actual limit is 127, not 255). Fixes #502 --- .../archive/crawler/frontier/BdbMultipleWorkQueues.java | 8 +++++++- modules/src/main/java/org/archive/modules/CrawlURI.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java index fabd685f..baebb909 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java +++ b/engine/src/main/java/org/archive/crawler/frontier/BdbMultipleWorkQueues.java @@ -434,7 +434,13 @@ public class BdbMultipleWorkQueues { keyData[len] = 0; long ordinalPlus = curi.getOrdinal() & 0x0000FFFFFFFFFFFFL; ordinalPlus = ((long) curi.getSchedulingDirective() << 56) | ordinalPlus; - long precedence = Math.min(curi.getPrecedence(), 127); + long rawPrecedence = curi.getPrecedence(); + if (rawPrecedence > 127) { + LOGGER.warning("URI precedence " + rawPrecedence + + " exceeds maximum 127 and will be clipped" + + " for queue ordering: " + curi); + } + long precedence = Math.min(rawPrecedence, 127); ordinalPlus = (((precedence) & 0xFFL) << 48) | ordinalPlus; ArchiveUtils.longIntoByteArray(ordinalPlus, keyData, len + 1); return new DatabaseEntry(keyData); diff --git a/modules/src/main/java/org/archive/modules/CrawlURI.java b/modules/src/main/java/org/archive/modules/CrawlURI.java index 15ffc996..624572ca 100644 --- a/modules/src/main/java/org/archive/modules/CrawlURI.java +++ b/modules/src/main/java/org/archive/modules/CrawlURI.java @@ -997,7 +997,7 @@ implements Reporter, Serializable, OverlayContext, Comparable { } /** spot for an integer cost to be placed by external facility (frontier). - * cost is truncated to 8 bits at times, so should not exceed 255 */ + * cost is truncated to 7 bits at times, so should not exceed 127 */ protected int holderCost = UNCALCULATED; /** * Return the 'holderCost' for convenience of external facility (frontier)