From 9f77412581bf655e7e69153feaa5284993d8f798 Mon Sep 17 00:00:00 2001 From: szznax Date: Tue, 23 Mar 2010 20:52:40 +0000 Subject: [PATCH] part of fix for HER-1627 incorrect source tag in crawl.log (of persist-loaded crawl) * EnhancedEnvironment.java in getClassCatalog(), when read-only env, insist on read-only catalog * PersistLoadProcessor.java remove existing curi members from prior to prevent clobbering source * PersistProcessor.java set historyDbConfig read-only, set je.log.faultReadSize for compatability with BDBModule.setup() --- .../java/org/archive/util/bdbje/EnhancedEnvironment.java | 1 + .../org/archive/modules/recrawl/PersistLoadProcessor.java | 1 + .../java/org/archive/modules/recrawl/PersistProcessor.java | 6 +++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java b/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java index cb28ce58..f9c974ba 100644 --- a/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java +++ b/commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java @@ -59,6 +59,7 @@ public class EnhancedEnvironment extends Environment { if(classCatalog == null) { DatabaseConfig dbConfig = new DatabaseConfig(); dbConfig.setAllowCreate(true); + dbConfig.setReadOnly(this.getConfig().getReadOnly()); try { classCatalogDB = openDatabase(null, "classCatalog", dbConfig); classCatalog = new StoredClassCatalog(classCatalogDB); diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java index 9de2cb48..841ba858 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java @@ -74,6 +74,7 @@ public class PersistLoadProcessor extends PersistOnlineProcessor { (Map) store.get(pkey); if(prior!=null) { // merge in keys + prior.keySet().removeAll(curi.getData().keySet()); curi.getData().putAll(prior); } } diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java index 05cd5853..b422e0ad 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java @@ -52,6 +52,7 @@ import com.sleepycat.bind.tuple.StringBinding; import com.sleepycat.collections.StoredIterator; import com.sleepycat.collections.StoredSortedMap; import com.sleepycat.je.Database; +import com.sleepycat.je.DatabaseConfig; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.EnvironmentConfig; @@ -142,8 +143,10 @@ public abstract class PersistProcessor extends Processor { // open the source env history DB, copying entries to target env EnhancedEnvironment sourceEnv = setupEnvironment(sourceDir, true); StoredClassCatalog sourceClassCatalog = sourceEnv.getClassCatalog(); + DatabaseConfig historyDbConfig = HISTORY_DB_CONFIG.toDatabaseConfig(); + historyDbConfig.setReadOnly(true); Database sourceHistoryDB = sourceEnv.openDatabase( - null, URI_HISTORY_DBNAME, HISTORY_DB_CONFIG.toDatabaseConfig()); + null, URI_HISTORY_DBNAME, historyDbConfig); StoredSortedMap sourceHistoryMap = new StoredSortedMap(sourceHistoryDB, new StringBinding(), new SerialBinding(sourceClassCatalog, Map.class), true); @@ -351,6 +354,7 @@ public abstract class PersistProcessor extends Processor { EnvironmentConfig envConfig = new EnvironmentConfig(); envConfig.setAllowCreate(true); envConfig.setReadOnly(readOnly); + envConfig.setConfigParam("je.log.faultReadSize", "6144"); return new EnhancedEnvironment(env, envConfig); } } \ No newline at end of file