mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-15 18:34:03 +00:00
[HER-1607] handle persist state when crawl starts up
* PersistProcessor.java, PersistLoadProcessor.java
port [HER-1586] preloadSource setting/refactoring to H3
* IoUtils.java
(getBufferedReader) utility method
* BdbModule.java
explicit passthrough of settable deferredWrite property
This commit is contained in:
1 parent
beb858369b
commit
608a3ddc8b
4 files changed
+276
-163
No files matched your search
@@ -102,7 +102,7 @@ Serializable, Closeable {
|
||||
boolean allowCreate;
|
||||
boolean sortedDuplicates;
|
||||
boolean transactional;
|
||||
|
||||
boolean deferredWrite = true;
|
||||
|
||||
public BdbConfig() {
|
||||
}
|
||||
@@ -129,7 +129,7 @@ Serializable, Closeable {
|
||||
|
||||
public DatabaseConfig toDatabaseConfig() {
|
||||
DatabaseConfig result = new DatabaseConfig();
|
||||
result.setDeferredWrite(true);
|
||||
result.setDeferredWrite(deferredWrite);
|
||||
result.setTransactional(transactional);
|
||||
result.setAllowCreate(allowCreate);
|
||||
result.setSortedDuplicates(sortedDuplicates);
|
||||
@@ -145,6 +145,11 @@ Serializable, Closeable {
|
||||
public void setTransactional(boolean transactional) {
|
||||
this.transactional = transactional;
|
||||
}
|
||||
|
||||
|
||||
public void setDeferredWrite(boolean b) {
|
||||
this.deferredWrite = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -37,6 +37,8 @@ import java.io.ObjectOutputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.io.Reader;
|
||||
import java.io.Serializable;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
@@ -352,4 +354,22 @@ public class IoUtils {
|
||||
}
|
||||
return new BufferedReader(new InputStreamReader(is));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a BufferedReader on the crawler journal given.
|
||||
*
|
||||
* @param source URL journal
|
||||
* @return journal buffered reader.
|
||||
* @throws IOException
|
||||
*/
|
||||
public static BufferedReader getBufferedReader(URL source) throws IOException {
|
||||
URLConnection conn = source.openConnection();
|
||||
boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip")
|
||||
|| conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip");
|
||||
InputStream uis = conn.getInputStream();
|
||||
return new BufferedReader(isGzipped?
|
||||
new InputStreamReader(new GZIPInputStream(uis)):
|
||||
new InputStreamReader(uis));
|
||||
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user