diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java new file mode 100644 index 00000000..f324d946 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ObjectIdentityBdbCache.java @@ -0,0 +1,564 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.Closeable; +import java.io.File; +import java.io.Serializable; +import java.lang.ref.PhantomReference; +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.SoftReference; +import java.lang.reflect.Field; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import java.util.logging.Logger; + +import com.sleepycat.bind.EntryBinding; +import com.sleepycat.bind.serial.SerialBinding; +import com.sleepycat.bind.serial.StoredClassCatalog; +import com.sleepycat.bind.tuple.TupleBinding; +import com.sleepycat.collections.StoredSortedMap; +import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseConfig; +import com.sleepycat.je.DatabaseException; +import com.sleepycat.je.Environment; + +/** + * A BDB JE backed object cache. + * + * Soft references to previously-instantiated objects are held so that + * unless/until an object is garbage collected, subsequent get()s will + * return the exact same object. (If all outside references are lost, + * when the soft reference is broken, the object state -- still + * accessible to this class via reflective access to a phantom + * referent --is flushed to disk. The next get() will reconsitute a new + * object, from the disk state.) + *

+ * The backing disk is only guaranteed to be up-to-date after a flush + * of all in-memory values to disk, as can be forced by sync(). + *

+ * To ensure that changes/mutations to values in this map are coherent and + * consistent at the application level, it is assumed that the application + * level only mutates values that are in this map and does not retain references + * to values longer than necessary. This allows mappings to be persisted + * during GC without explicit transactions or write operations. + *

+ * Based on the earlier CachedBdbMap. + *

+ * + * @author John Erik Halse + * @author stack + * @author gojomo + * @author paul baclace (conversion to ConcurrentMap) + * + */ +public class ObjectIdentityBdbCache +implements ObjectIdentityCache, Closeable, Serializable { + private static final long serialVersionUID = 1L; + + private static final Logger logger = + Logger.getLogger(ObjectIdentityBdbCache.class.getName()); + + /** The BDB JE database used for this instance. */ + protected transient Database db; + + /** in-memory map of new/recent/still-referenced-elsewhere instances */ + protected transient ConcurrentHashMap> memMap; + protected transient ReferenceQueue refQueue; + + /** The Collection view of the BDB JE database used for this instance. */ + protected transient StoredSortedMap diskMap; + + // + // USAGE STATS + // + /** Count of times we got an object from in-memory cache */ + private AtomicLong cacheHit = new AtomicLong(0); + /** Count of times the {@link ObjectIdentityBdbCache#get} method was called. */ + private AtomicLong countOfGets = new AtomicLong(0); + /** Count of every time disk-based map provided non-null object */ + private AtomicLong diskHit = new AtomicLong(0); + /** Count of times Supplier was used for new object */ + private AtomicLong supplierUsed = new AtomicLong(0); + /** count of expunge put() to BDB (implies disk) */ + private AtomicLong expungeStatsDiskPut = new AtomicLong(0); + /** count of {@link #sync()} use */ + transient private AtomicLong useStatsSyncUsed = new AtomicLong(0); + + /** Reference to the Reference#referent Field. */ + protected static Field referentField; + static { + // We need access to the referent field in the PhantomReference. + // For more on this trick, see + // + // http://www.javaspecialists.co.za/archive/Issue098.html and for + // discussion: + // http://www.theserverside.com/tss?service=direct/0/NewsThread/threadViewer.markNoisy.link&sp=l29865&sp=l146901 + try { + referentField = Reference.class.getDeclaredField("referent"); + referentField.setAccessible(true); + } catch (SecurityException e) { + throw new RuntimeException(e); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + + /** + * Constructor. You must call + * {@link #initialize(Environment, Class, Class, StoredClassCatalog)} + * to finish construction. Construction is two-stepped to support + * reconnecting a deserialized CachedBdbMap with its backing bdbje + * database. + * + * @param dbName Name of the backing db this instance should use. + */ + public ObjectIdentityBdbCache() { + super(); + } + + /** + * Call this method when you have an instance when you used the + * default constructor or when you have a deserialized instance that you + * want to reconnect with an extant bdbje environment. Do not + * call this method if you used the + * {@link #CachedBdbMap(File, String, Class, Class)} constructor. + * @param env + * @param keyClass + * @param valueClass + * @param classCatalog + * @throws DatabaseException + */ + public synchronized void initialize(final Environment env, String dbName, + final Class valueClass, final StoredClassCatalog classCatalog) + throws DatabaseException { + // TODO: initial capacity should be related to number of seeds, max depth, max docs + this.memMap = new ConcurrentHashMap>( + 8192, // initial capacity + 0.9f, // acceptable load factor + 64 // est. number of concurrent threads + ); + this.refQueue = new ReferenceQueue(); + canary = new SoftReference(new LowMemoryCanary()); + + this.db = openDatabase(env, dbName); + this.diskMap = createDiskMap(this.db, classCatalog, valueClass); + } + + @SuppressWarnings("unchecked") + protected StoredSortedMap createDiskMap(Database database, + StoredClassCatalog classCatalog, Class valueClass) { + EntryBinding keyBinding = TupleBinding.getPrimitiveBinding(String.class); + EntryBinding valueBinding = TupleBinding.getPrimitiveBinding(valueClass); + if(valueBinding == null) { + valueBinding = new SerialBinding(classCatalog, valueClass); + } + return new StoredSortedMap(database, keyBinding, valueBinding, true); + } + + protected Database openDatabase(final Environment environment, + final String dbName) throws DatabaseException { + DatabaseConfig dbConfig = new DatabaseConfig(); + dbConfig.setTransactional(false); + dbConfig.setAllowCreate(true); + dbConfig.setDeferredWrite(true); + return environment.openDatabase(null, dbName, dbConfig); + } + + /* (non-Javadoc) + * @see org.archive.util.ObjectIdentityCache#close() + */ + public synchronized void close() { + // Close out my bdb db. + if (this.db != null) { + try { + sync(); + this.db.sync(); + this.db.close(); + } catch (DatabaseException e) { + logger.log(Level.WARNING,"problem closing ObjectIdentityBdbCache",e); + } finally { + this.db = null; + } + } + } + + protected void finalize() throws Throwable { + close(); + super.finalize(); + } + + /* (non-Javadoc) + * @see org.archive.util.ObjectIdentityCache#get(java.lang.String) + */ + public V get(final String key) { + return getOrUse(key,null); + } + + /* (non-Javadoc) + * @see org.archive.util.ObjectIdentityCache#get(java.lang.String, org.archive.util.ObjectIdentityBdbCache) + */ + public V getOrUse(final String key, Supplier supplierOrNull) { + countOfGets.incrementAndGet(); + + if (countOfGets.get() % 10000 == 0) { + logCacheSummary(); + } + + // check mem cache + SoftEntry entry = memMap.get(key); + if(entry != null) { + V val = entry.get(); + if(val != null) { + // the concurrent garden path: in mem, valid + cacheHit.incrementAndGet(); + return val; + } + } + + // everything other difficult case happens inside this block + synchronized(this) { + // recheck mem cache -- if another thread beat us into sync + // block and already filled the key + entry = memMap.get(key); + if(entry != null) { + V val = entry.get(); + if(val != null) { + cacheHit.incrementAndGet(); + return val; + } + } + // persist to disk all stale (soft-ref-cleared) entries now + pageOutStaleEntries(); + + // check disk + V valDisk = (V) diskMap.get(key); + if(valDisk==null) { + // never yet created, consider creating + if(supplierOrNull==null) { + return null; + } + // create using provided Supplier + valDisk = supplierOrNull.get(); + supplierUsed.incrementAndGet(); + V prevVal = diskMap.putIfAbsent(key, valDisk); + if(prevVal!=null) { + // ERROR: diskMap modification since previous + // diskMap.get() should be impossible + logger.log(Level.SEVERE,"diskMap modified outside synchronized block?"); + } + } else { + diskHit.incrementAndGet(); + } + + // keep new val in memMap + SoftEntry newEntry = new SoftEntry(key, valDisk, refQueue); + SoftEntry prevVal = memMap.putIfAbsent(key, newEntry); + if(prevVal != null) { + // ERROR: memMap modification since previous + // memMap.get() should be impossible + logger.log(Level.SEVERE,"memMap modified outside synchronized block?"); + } + if(valDisk==null) { + System.err.println("ruh-roh!"); + } + return valDisk; + } + } + + public Set keySet() { + return diskMap.keySet(); + } + + /** + * Info to log, if at FINE level + */ + private void logCacheSummary() { + if (logger.isLoggable((Level.FINE))) { + logger.fine(composeCacheSummary()); + } + } + + private String composeCacheSummary() { + long totalHits = cacheHit.get() + diskHit.get(); + if (totalHits < 1) { + return ""; + } + long cacheHitPercent + = (cacheHit.get() * 100) / totalHits; + StringBuilder sb = new StringBuilder(120); + sb.append("DB name:") + .append(getDatabaseName()) + .append(", ") + .append(" hit%: ") + .append(cacheHitPercent) + .append("%, gets=") + .append(countOfGets.get()) + .append(" memHits=") + .append(cacheHit.get()) + .append(" diskHits=") + .append(diskHit.get()) + .append(" supplieds=") + .append(supplierUsed.get()) + .append(" expungePuts=") + .append(expungeStatsDiskPut.get()) + .append(" syncs=") + .append(useStatsSyncUsed.get()); + return sb.toString(); + } + + /* (non-Javadoc) + * @see org.archive.util.ObjectIdentityCache#size() + */ + public int size() { + return diskMap.size(); + } + + protected String getDatabaseName() { + String name = "DbName-Lookup-Failed"; + try { + if (this.db != null) { + name = this.db.getDatabaseName(); + } + } catch (DatabaseException e) { + // Ignore. + } + return name; + } + + /** + * Sync all in-memory map entries to backing disk store. + */ + @SuppressWarnings("unchecked") + public synchronized void sync() { + String dbName = null; + // Sync. memory and disk. + useStatsSyncUsed.incrementAndGet(); + long startTime = 0; + if (logger.isLoggable(Level.INFO)) { + dbName = getDatabaseName(); + startTime = System.currentTimeMillis(); + logger.info(dbName + " start sizes: disk " + this.diskMap.size() + + ", mem " + this.memMap.size()); + } + + for (String key : this.memMap.keySet()) { + SoftEntry entry = memMap.get(key); + if (entry != null) { + // Get & hold so not cleared pre-return. + V value = entry.get(); + if (value != null) { + expungeStatsDiskPut.incrementAndGet(); + this.diskMap.put(key, value); // unchecked cast + } + } + } + pageOutStaleEntries(); + + // force sync of deferred-writes + try { + this.db.sync(); + } catch (DatabaseException e) { + throw new RuntimeException(e); + } + + if (logger.isLoggable(Level.INFO)) { + logger.info(dbName + " sync took " + + (System.currentTimeMillis() - startTime) + "ms. " + + "Finish sizes: disk " + + this.diskMap.size() + ", mem " + this.memMap.size()); + } + } + + /** An incremental, poll-based expunger. + */ + @SuppressWarnings("unchecked") + private synchronized void pageOutStaleEntries() { + int c = 0; + long startTime = System.currentTimeMillis(); + for(SoftEntry entry; (entry = (SoftEntry)refQueue.poll()) != null;) { + pageOutStaleEntry(entry); + c++; + } + if (c > 0 && logger.isLoggable(Level.FINER)) { + long endTime = System.currentTimeMillis(); + try { + logger.finer("DB: " + db.getDatabaseName() + ", Expunged: " + + c + ", Diskmap size: " + diskMap.size() + + ", Cache size: " + memMap.size() + + ", in "+(endTime-startTime)+"ms"); + } catch (DatabaseException e) { + logger.log(Level.FINER,"exception while logging",e); + } + } + } + + /** + * Expunge an entry from memMap while updating diskMap. + * + * @param entry a SoftEntry obtained from refQueuePoll() + */ + private void pageOutStaleEntry(SoftEntry entry) { + V phantomValue = null; + // keep this ref on stack so it cannot be nulled in mid-operation + PhantomEntry phantom = entry.getPhantom(); + if (phantom == null) { + logger.log(Level.WARNING,"unexpected null phantom before flush", new Exception()); + return; // nothing to do + } else { + // recover hidden value + phantomValue = phantom.doctoredGet(); + } + + // Still in memMap? (should be) + if (memMap.get(phantom.getKey()) != entry) { // NOTE: intentional identity compare + logger.log(Level.WARNING,"unexpected memMap mismatch", new Exception()); + return; // nothing to do + } + + // Expected value present? (should be) + if(phantomValue == null) { + logger.log(Level.WARNING,"unexpected null phantomValue", new Exception()); + return; // nothing to do + } + + // given instance entry still in memMap; + // we have the key and if we have a phantom Value, then + // the diskMap can be updated. + diskMap.put(phantom.getKey(), phantomValue); // unchecked cast + expungeStatsDiskPut.incrementAndGet(); + + // remove memMap entry that matches key and value; + // since only this thread has the exclusive expunge job + // for entry, we will either remove the exact object + // or remove no object (non-interference). + boolean removed = memMap.remove(phantom.getKey(), entry); + if(!removed) { + logger.log(Level.WARNING,"expunge memMap.remove() ineffective",new Exception()); + } + entry.clearPhantom(); // truly allows GC of unreferenced V object + } + + private static class PhantomEntry extends PhantomReference { + private final String key; + + public PhantomEntry(String key, V referent) { + super(referent, null); + this.key = key; + } + + /** + * @return Return the referent. The contract for {@link #get()} + * always returns a null referent. We've cheated and doctored + * PhantomReference to return the actual referent value. See notes + * at {@link #referentField}; + */ + @SuppressWarnings("unchecked") + final public V doctoredGet() { + try { + // Here we use the referentField saved off on static + // initialization of this class to get at this References' + // private referent field. + return (V) referentField.get(this); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + /** + * @return Returns the key. + */ + final public String getKey() { + return this.key; + } + } + + /** + * SoftReference cache entry. + * + * A PhantomReference is used to hold the key and value as a last + * chance before GC hook that can effect the update of diskMap. + *

+ * Entries are not recycled. + */ + private static class SoftEntry extends SoftReference { + private PhantomEntry phantom; + + @SuppressWarnings("unchecked") + public SoftEntry(String key, V referent, ReferenceQueue q) { + super(referent, q); + this.phantom = new PhantomEntry(key, referent); // unchecked cast + } + + public V get() { + // ensure visibility + synchronized (this) { + return super.get(); + } + } + + /** + * @return Returns the phantom reference. + */ + final public PhantomEntry getPhantom() { + return this.phantom; + } + + /** + * Clear referent and end-of-life-cycle this instance. Idempotent. + */ + final void clearPhantom() { + if (this.phantom != null) { + this.phantom.clear(); + this.phantom = null; + } + super.clear(); + } + + public String toString() { + if (phantom != null) { + return "SoftEntry(key=" + phantom.getKey().toString() + ")"; + } else { + return "SoftEntry()"; + } + } + } + + // + // Crude, probably unreliable/fragile but harmless mechanism to + // trigger expunge of cleared SoftReferences in low-memory + // conditions even without any of the other get/put triggers. + // + + protected transient SoftReference canary; + protected class LowMemoryCanary { + /** When collected/finalized -- as should be expected in + * low-memory conditions -- trigger an expunge and a + * new 'canary' insertion. */ + public void finalize() { + ObjectIdentityBdbCache.this.pageOutStaleEntries(); + ObjectIdentityBdbCache.this.canary = + new SoftReference(new LowMemoryCanary()); + } + } +} diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityCache.java new file mode 100644 index 00000000..564cd995 --- /dev/null +++ b/commons/src/main/java/org/archive/util/ObjectIdentityCache.java @@ -0,0 +1,62 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.io.Closeable; +import java.util.Set; + +/** + * An object cache for create-once-by-name-and-then-reuse objects. + * + * Objects are added, but never removed. Subsequent get()s using the + * same key will return the exact same object, UNLESS all such objects + * have been forgotten, in which case a new object MAY be returned. + * + * This allows implementors (such as ObjectIdentityBdbCache or + * CachedBdbMap) to page out (aka 'expunge') instances to + * persistent storage while they're not being used. However, as long as + * they are used (referenced), all requests for the same-named object + * will share a reference to the same object, and the object may be + * mutated in place without concern for explicitly persisting its + * state to disk. + * + * @param + */ +public interface ObjectIdentityCache extends Closeable { + /** get the object under the given key/name */ + public abstract V get(final K key); + + /** get the object under the given key/name, using (and remembering) + * the object supplied by the supplier if no prior mapping exists */ + public abstract V getOrUse(final K key, Supplier supplierOrNull); + + /** force the persistent backend, if any, to be updated with all + * live object state */ + public abstract void sync(); + + /** close/release any associated resources */ + public abstract void close(); + + /** count of name-to-object contained */ + public abstract int size(); + + /** set of all keys */ + public abstract Set keySet(); +} \ No newline at end of file diff --git a/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java b/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java new file mode 100644 index 00000000..c6fc329e --- /dev/null +++ b/commons/src/main/java/org/archive/util/ObjectIdentityMemCache.java @@ -0,0 +1,75 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.archive.util; + +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Trivial all-in-memory object cache, using a single internal + * ConcurrentHashMap. + * + * @contributor gojomo + * @param + */ +public class ObjectIdentityMemCache +implements ObjectIdentityCache { + ConcurrentHashMap map; + + public ObjectIdentityMemCache() { + map = new ConcurrentHashMap(); + } + + public ObjectIdentityMemCache(int cap, float load, int conc) { + map = new ConcurrentHashMap(cap, load, conc); + } + + public void close() { + // do nothing + } + + public V get(String key) { + return map.get(key); + } + + public V getOrUse(String key, Supplier supplierOrNull) { + V val = map.get(key); + if (val==null && supplierOrNull!=null) { + val = supplierOrNull.get(); + V prevVal = map.putIfAbsent(key, val); + if(prevVal!=null) { + val = prevVal; + } + } + return val; + } + + public int size() { + return map.size(); + } + + public Set keySet() { + return map.keySet(); + } + + public void sync() { + // do nothing + } +} diff --git a/commons/src/main/java/org/archive/util/Supplier.java b/commons/src/main/java/org/archive/util/Supplier.java new file mode 100644 index 00000000..16a68ae6 --- /dev/null +++ b/commons/src/main/java/org/archive/util/Supplier.java @@ -0,0 +1,43 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.util; + +/** + * Class for optionally providing one instance of the parameterized + * type. The one instance may be provided at construction, or by + * overriding get() in subclasses, may be created on demand. + * + * @param + */ +public class Supplier { + V instance; + + public Supplier() { + super(); + } + + public Supplier(V instance) { + super(); + this.instance = instance; + } + + public V get() { + return instance; + } +} \ No newline at end of file diff --git a/commons/src/test/java/org/archive/util/ObjectIdentityBdbCacheTest.java b/commons/src/test/java/org/archive/util/ObjectIdentityBdbCacheTest.java new file mode 100644 index 00000000..325003d5 --- /dev/null +++ b/commons/src/test/java/org/archive/util/ObjectIdentityBdbCacheTest.java @@ -0,0 +1,195 @@ +/* ObjectIdentityBdbCacheTest + * + * Copyright (C) 2009 Internet Archive. + * + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Heritrix is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser Public License as published by + * the Free Software Foundation; either version 2.1 of the License, or + * any later version. + * + * Heritrix is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser Public License for more details. + * + * You should have received a copy of the GNU Lesser Public License + * along with Heritrix; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +package org.archive.util; + +import java.io.File; +import java.util.HashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.lang.math.RandomUtils; +import org.archive.util.bdbje.EnhancedEnvironment; + +import org.apache.commons.io.FileUtils; + +/** + * @contributor stack + * @contributor gojomo + * @version $Date: 2009-08-03 23:50:43 -0700 (Mon, 03 Aug 2009) $, $Revision: 6434 $ + */ +public class ObjectIdentityBdbCacheTest extends TmpDirTestCase { + EnhancedEnvironment env; + private ObjectIdentityBdbCache> cache; + + @SuppressWarnings("unchecked") + protected void setUp() throws Exception { + super.setUp(); + File envDir = new File(getTmpDir(),"ObjectIdentityBdbCacheTest"); + envDir.mkdirs(); + FileUtils.deleteDirectory(envDir); + envDir.mkdirs(); + env = EnhancedEnvironment.getTestEnvironment(envDir); + this.cache = new ObjectIdentityBdbCache>(); + this.cache.initialize(env,"setUpCache",HashMap.class, env.getClassCatalog()); + } + + protected void tearDown() throws Exception { + this.cache.close(); + File envDir = env.getHome(); + env.close(); + FileUtils.deleteDirectory(envDir); + super.tearDown(); + } + + @SuppressWarnings("unchecked") + public void testReadConsistencyUnderLoad() throws Exception { + final ObjectIdentityBdbCache cbdbmap = + new ObjectIdentityBdbCache(); + cbdbmap.initialize(env, + "consistencyCache", + AtomicInteger.class, + env.getClassCatalog()); + try { + final AtomicInteger level = new AtomicInteger(0); + final int keyCount = 128 * 1024; // 128K keys + final int maxLevel = 64; + // initial fill + for(int i=0; i < keyCount; i++) { + cbdbmap.getOrUse(""+i, new Supplier(new AtomicInteger(level.get()))); + } + // backward checking that all values always at level or higher + new Thread() { + public void run() { + untilmax: while(true) { + for(int j=keyCount-1; j >= 0; j--) { + int targetValue = level.get(); + if(targetValue>=maxLevel) { + break untilmax; + } + assertTrue("stale value revseq key "+j,cbdbmap.get(""+j).get()>=targetValue); + Thread.yield(); + } + } + } + }.start(); + // random checking that all values always at level or higher + new Thread() { + public void run() { + untilmax: while(true) { + int j = RandomUtils.nextInt(keyCount); + int targetValue = level.get(); + if(targetValue>=maxLevel) { + break untilmax; + } + assertTrue("stale value random key "+j, + cbdbmap.get(""+j).get()>=targetValue); + Thread.yield(); + } + } + }.start(); + // increment all keys + for(; level.get() < maxLevel; level.incrementAndGet()) { + for(int k = 0; k < keyCount; k++) { + int foundValue = cbdbmap.get(""+k).getAndIncrement(); + assertEquals("stale value preinc key "+k, level.get(), foundValue); + } + if(level.get() % 10 == 0) { + System.out.println("level to "+level.get()); + } + Thread.yield(); + } + } finally { + cbdbmap.close(); + } + // SUCCESS + } + + public void testBackingDbGetsUpdated() { + // Enable all logging. Up the level on the handlers and then + // on the big map itself. + Handler [] handlers = Logger.getLogger("").getHandlers(); + for (int index = 0; index < handlers.length; index++) { + handlers[index].setLevel(Level.FINEST); + } + Logger.getLogger(ObjectIdentityBdbCache.class.getName()). + setLevel(Level.FINEST); + // Set up values. + final String value = "value"; + final String key = "key"; + final int upperbound = 3; + // First put in empty hashmap. + for (int i = 0; i < upperbound; i++) { + this.cache.getOrUse(key + Integer.toString(i), new Supplier>(new HashMap())); + } + // Now add value to hash map. + for (int i = 0; i < upperbound; i++) { + HashMap m = this.cache.get(key + Integer.toString(i)); + m.put(key, value); + } + this.cache.sync(); + for (int i = 0; i < upperbound; i++) { + HashMap m = this.cache.get(key + Integer.toString(i)); + String v = m.get(key); + if (v == null || !v.equals(value)) { + Logger.getLogger(CachedBdbMap.class.getName()). + warning("Wrong value " + i); + } + } + } + + /** + * Test that in scarce memory conditions, the memory map is + * expunged of otherwise unreferenced entries as expected. + * + * NOTE: this test may be especially fragile with regard to + * GC/timing issues; relies on timely finalization, which is + * never guaranteed by JVM/GC. For example, it is so sensitive + * to CPU speed that a Thread.sleep(1000) succeeds when my + * laptop is plugged in, but fails when it is on battery! + * + * @throws InterruptedException + */ + public void testMemMapCleared() throws InterruptedException { + TestUtils.forceScarceMemory(); + System.gc(); // minimize effects of earlier test heap use + assertEquals(0, cache.memMap.size()); + assertEquals(0, cache.diskMap.size()); + for(int i=0; i < 10000; i++) { + cache.getOrUse(""+i, new Supplier>(new HashMap())); + } + assertEquals(cache.memMap.size(), 10000); + assertEquals(cache.size(), 10000); + TestUtils.forceScarceMemory(); + Thread.sleep(3000); + // The 'canary' trick makes this explicit expunge, or + // an expunge triggered by a get() or put...(), unnecessary + // cache.expungeStaleEntries(); + System.out.println(cache.size()+","+cache.memMap.size()); + assertEquals(0, cache.memMap.size()); + } + + + public static void main(String [] args) { + junit.textui.TestRunner.run(ObjectIdentityBdbCacheTest.class); + } +}