[HER-1607] handle persist state when crawl starts up

* PersistProcessor.java, PersistLoadProcessor.java
    port [HER-1586] preloadSource setting/refactoring to H3
* IoUtils.java
    (getBufferedReader) utility method
* BdbModule.java
    explicit passthrough of settable deferredWrite property
This commit is contained in:
gojomo committed 2009-08-27 01:32:08 +00:00
1 parent beb858369b
commit 608a3ddc8b
4 files changed
+276 -163

No files matched your search

@@ -102,7 +102,7 @@ Serializable, Closeable {
boolean allowCreate;
boolean sortedDuplicates;
boolean transactional;
boolean deferredWrite = true;
public BdbConfig() {
}
@@ -129,7 +129,7 @@ Serializable, Closeable {
public DatabaseConfig toDatabaseConfig() {
DatabaseConfig result = new DatabaseConfig();
result.setDeferredWrite(true);
result.setDeferredWrite(deferredWrite);
result.setTransactional(transactional);
result.setAllowCreate(allowCreate);
result.setSortedDuplicates(sortedDuplicates);
@@ -145,6 +145,11 @@ Serializable, Closeable {
public void setTransactional(boolean transactional) {
this.transactional = transactional;
}
public void setDeferredWrite(boolean b) {
this.deferredWrite = true;
}
}
@@ -37,6 +37,8 @@ import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Reader;
import java.io.Serializable;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.util.Iterator;
import java.util.List;
@@ -352,4 +354,22 @@ public class IoUtils {
}
return new BufferedReader(new InputStreamReader(is));
}
/**
* Get a BufferedReader on the crawler journal given.
*
* @param source URL journal
* @return journal buffered reader.
* @throws IOException
*/
public static BufferedReader getBufferedReader(URL source) throws IOException {
URLConnection conn = source.openConnection();
boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip")
|| conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip");
InputStream uis = conn.getInputStream();
return new BufferedReader(isGzipped?
new InputStreamReader(new GZIPInputStream(uis)):
new InputStreamReader(uis));
}
}