lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-23 14:15:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #395 from internetarchive/adds-trough-dedup-performance-rework

Browse Source 
Refactor Trough client URL cache put. Limit TroughContentDigestHistor…
This commit is contained in:
Adam Miller
2021-05-28 16:15:22 -07:00
committed by GitHub
parent 8c59cdfbad 58bf0cf45c
commit c403d7b913
2 changed files with 29 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/src/main/java/org/archive/modules/recrawl/TroughContentDigestHistory.java
+1 -1
View File
@@ -197,7 +197,7 @@ public class TroughContentDigestHistory extends AbstractContentDigestHistory imp
}
if(!memoryDedupHit) {
try {
String sql = "select * from dedup where digest_key = %s";
String sql = "select * from dedup where digest_key = %s limit 1";
List<Map<String, Object>> results = troughClient().read(getSegmentId(), sql, new String[]{persistKeyFor(curi)});
if (!results.isEmpty()) {
Map<String, Object> hist = new HashMap<String, Object>();
contrib/src/main/java/org/archive/trough/TroughClient.java
+28 -5
View File
@@ -318,8 +318,20 @@ public class TroughClient {
protected String readUrl(String segmentId) throws TroughException {
if (readUrlCache.get(segmentId) == null) {
String url = readUrlNoCache(segmentId);
readUrlCache.put(segmentId, url);
try {
readUrlCache.computeIfAbsent(segmentId, k -> {
try {
return readUrlNoCache(k);
} catch (TroughException e) {
throw new RuntimeException(e);
}
});
}
catch(RuntimeException e) {
if(e.getCause() instanceof TroughException){
throw (TroughException)e.getCause();
}
}
logger.info("segment " + segmentId + " read url is " + readUrlCache.get(segmentId));
}
return readUrlCache.get(segmentId);
@@ -379,9 +391,20 @@ public class TroughClient {
protected String writeUrl(String segmentId, String schemaId) throws IOException {
if (writeUrlCache.get(segmentId) == null) {
String url = writeUrlNoCache(segmentId, schemaId);
writeUrlCache.put(segmentId, url);
try {
writeUrlCache.computeIfAbsent(segmentId, k -> {
try {
return writeUrlNoCache(k, schemaId);
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
catch(RuntimeException e) {
if(e.getCause() instanceof IOException){
throw (IOException)e.getCause();
}
}
logger.info("segment " + segmentId + " write url is " + writeUrlCache.get(segmentId));
}
return writeUrlCache.get(segmentId);