mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-10-06 12:21:27 +00:00
[HER-1658] CachedBdbMaps not expunging as expected (especially StatisticsTracker.processedSeedsRecords)
* CachedBdbMap.java
do expunge on put(), replace()
add low-memory-sensitive 'canary' to force expunge even if otherwise untriggered
* CachedBdbMapTest.java
add test of idle expunge in low-memory conditions
* CrawlURI.java
(getURI) direct access to String URI (don't reuse toString() functionally)
* SeedRecord.java
support for updating record with later/repeat report
* StatisticsTracker.java
remove put()s for processedSeedRecords, hostsLastFinished
* BdbFrontier.java
(getQueueFor) remove put, do in concurrent-compliant manner
This commit is contained in:
1 parent
1c984753f0
commit
c196a1de77
6 files changed
+124
-84
No files matched your search
@@ -83,6 +83,32 @@ public class CachedBdbMapTest extends TmpDirTestCase {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that in scarce memory conditions, the memory map is
|
||||
* expunged of otherwise unreferenced entries as expected.
|
||||
*
|
||||
* NOTE: this test may be especially fragile with regard to
|
||||
* GC/timing issues; relies on timely finalization, which is
|
||||
* never guaranteed by JVM/GC.
|
||||
*
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public void testMemMapCleared() throws InterruptedException {
|
||||
assertEquals(cache.memMap.size(), 0);
|
||||
for(int i=0; i < 10000; i++) {
|
||||
cache.putIfAbsent(""+i, new HashMap<String,String>());
|
||||
}
|
||||
assertEquals(cache.memMap.size(), 10000);
|
||||
assertEquals(cache.size(), 10000);
|
||||
TestUtils.forceScarceMemory();
|
||||
Thread.sleep(1000);
|
||||
// The 'canary' trick makes this explicit expunge, or
|
||||
// an expunge triggered by a get() or put...(), unnecessary
|
||||
// cache.expungeStaleEntries();
|
||||
System.out.println(cache.size()+","+cache.memMap.size());
|
||||
assertEquals(0, cache.memMap.size());
|
||||
}
|
||||
|
||||
public static void main(String [] args) {
|
||||
junit.textui.TestRunner.run(CachedBdbMapTest.class);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user