lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 06:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

part of fix for HER-1627 incorrect source tag in crawl.log (of persist-loaded crawl)

Browse Source 
* EnhancedEnvironment.java
    in getClassCatalog(), when read-only env, insist on read-only catalog
* PersistLoadProcessor.java
    remove existing curi members from prior to prevent clobbering source
* PersistProcessor.java
    set historyDbConfig read-only, set je.log.faultReadSize for
        compatability with BDBModule.setup()
This commit is contained in:
szznax
2010-03-23 20:52:40 +00:00
parent 20b8a7bc90
commit 9f77412581
3 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
commons/src/main/java/org/archive/util/bdbje/EnhancedEnvironment.java
+1
View File
@@ -59,6 +59,7 @@ public class EnhancedEnvironment extends Environment {
if(classCatalog == null) {
DatabaseConfig dbConfig = new DatabaseConfig();
dbConfig.setAllowCreate(true);
dbConfig.setReadOnly(this.getConfig().getReadOnly());
try {
classCatalogDB = openDatabase(null, "classCatalog", dbConfig);
classCatalog = new StoredClassCatalog(classCatalogDB);
modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java
+1
View File
@@ -74,6 +74,7 @@ public class PersistLoadProcessor extends PersistOnlineProcessor {
(Map<String,Object>) store.get(pkey);
if(prior!=null) {
// merge in keys
prior.keySet().removeAll(curi.getData().keySet());
curi.getData().putAll(prior);
}
}
modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java
+5 -1
View File
@@ -52,6 +52,7 @@ import com.sleepycat.bind.tuple.StringBinding;
import com.sleepycat.collections.StoredIterator;
import com.sleepycat.collections.StoredSortedMap;
import com.sleepycat.je.Database;
import com.sleepycat.je.DatabaseConfig;
import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.EnvironmentConfig;
@@ -142,8 +143,10 @@ public abstract class PersistProcessor extends Processor {
// open the source env history DB, copying entries to target env
EnhancedEnvironment sourceEnv = setupEnvironment(sourceDir, true);
StoredClassCatalog sourceClassCatalog = sourceEnv.getClassCatalog();
DatabaseConfig historyDbConfig = HISTORY_DB_CONFIG.toDatabaseConfig();
historyDbConfig.setReadOnly(true);
Database sourceHistoryDB = sourceEnv.openDatabase(
null, URI_HISTORY_DBNAME, HISTORY_DB_CONFIG.toDatabaseConfig());
null, URI_HISTORY_DBNAME, historyDbConfig);
StoredSortedMap<String,Map> sourceHistoryMap = new StoredSortedMap<String,Map>(sourceHistoryDB,
new StringBinding(), new SerialBinding<Map>(sourceClassCatalog,
Map.class), true);
@@ -351,6 +354,7 @@ public abstract class PersistProcessor extends Processor {
EnvironmentConfig envConfig = new EnvironmentConfig();
envConfig.setAllowCreate(true);
envConfig.setReadOnly(readOnly);
envConfig.setConfigParam("je.log.faultReadSize", "6144");
return new EnhancedEnvironment(env, envConfig);
}
}