lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-27 08:05:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use simpler read 500k chars and split method suggested by @nlevitt

Browse Source 
While we temporarily use a little more memory this version is a lot
less codes. It also allows us to do away with the BufferedReader.
This commit is contained in:
Alex Osborne
2017-11-01 09:29:28 +09:00
parent 1fae77c64d
commit 68ceceedd2
3 changed files with 48 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/net/CrawlServer.java
+1 -2
View File
@@ -174,10 +174,9 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden
InputStream contentBodyStream = null;
try {
BufferedReader reader;
contentBodyStream = curi.getRecorder().getContentReplayInputStream();
reader = new BufferedReader(new InputStreamReader(contentBodyStream));
InputStreamReader reader = new InputStreamReader(contentBodyStream);
robotstxt = new Robotstxt(reader);
validRobots = true;
} catch (IOException e) {
modules/src/main/java/org/archive/modules/net/Robotstxt.java
+27 -81
View File
@@ -18,16 +18,17 @@
*/
package org.archive.modules.net;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.Serializable;
import java.nio.CharBuffer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.archive.bdb.AutoKryo;
@@ -43,7 +44,8 @@ public class Robotstxt implements Serializable {
private static final Logger logger =
Logger.getLogger(Robotstxt.class.getName());
protected static final long MAX_SIZE = 500*1024;
protected static final int MAX_SIZE = 500*1024;
private static final Pattern LINE_SEPARATOR = Pattern.compile("\r\n|\r|\n");
// all user agents contained in this robots.txt
// in order of declaration
@@ -63,12 +65,16 @@ public class Robotstxt implements Serializable {
public Robotstxt() {
}
public Robotstxt(BufferedReader reader) throws IOException {
initializeFromReader(reader);
public Robotstxt(Reader reader) throws IOException {
try {
initializeFromReader(reader);
} finally {
IOUtils.closeQuietly(reader);
}
}
public Robotstxt(ReadSource customRobots) {
BufferedReader reader = new BufferedReader(customRobots.obtainReader());
Reader reader = customRobots.obtainReader();
try {
initializeFromReader(reader);
} catch (IOException e) {
@@ -80,25 +86,24 @@ public class Robotstxt implements Serializable {
}
}
protected void initializeFromReader(BufferedReader reader) throws IOException {
BoundedLineReader lineReader = new BoundedLineReader(reader, MAX_SIZE);
String read;
protected void initializeFromReader(Reader reader) throws IOException {
CharBuffer buffer = CharBuffer.allocate(MAX_SIZE);
while (buffer.hasRemaining() && reader.read(buffer) >= 0) ;
buffer.flip();
String[] lines = LINE_SEPARATOR.split(buffer);
if (buffer.limit() == buffer.capacity()) {
int processed = buffer.capacity() - lines[lines.length - 1].length();
logger.warning("processed " + processed + " characters, ignoring the rest (see HER-1990)");
// discard the partial line at the end so we don't process a truncated path
lines[lines.length - 1] = "";
}
// current is the disallowed paths for the preceding User-Agent(s)
RobotsDirectives current = null;
while (reader != null) {
do {
read = lineReader.readLine();
// Skip comments & blanks
} while (read != null && ((read = read.trim()).startsWith("#") || read.length() == 0));
if (read == null) {
if (lineReader.reachedLimit()) {
// we count characters instead of bytes because the byte count isn't easily available
logger.warning("processed " + lineReader.getCharsProcessed() +
" characters, ignoring the rest (see HER-1990)");
}
reader.close();
reader = null;
} else {
for (String read: lines) {
read = read.trim();
if (!read.isEmpty() && !read.startsWith("#")) {
// remove any html markup
read = read.replaceAll("<[^>]+>","");
int commentIndex = read.indexOf("#");
@@ -188,65 +193,6 @@ public class Robotstxt implements Serializable {
}
}
/**
* Read lines from a reader until a character limit is reached.
*
* Always returns whole lines. If the limit would cause a partial line to be
* read the data is discarded.
*/
private static class BoundedLineReader {
private final Reader reader;
private long remaining;
private long charsProcessed = 0;
BoundedLineReader(Reader reader, long limit) {
this.reader = reader;
this.remaining = limit;
}
String readLine() throws IOException {
StringBuilder buffer = new StringBuilder();
while (!reachedLimit()) {
int c = reader.read();
if (c < 0) { // end of file
if (buffer.length() > 0) { // file didn't end on a linefeed
charsProcessed += buffer.length();
return buffer.toString();
} else {
return null;
}
}
remaining--;
if (c == '\r' || c == '\n') {
charsProcessed += buffer.length() + 1;
return buffer.toString();
}
buffer.append((char) c);
}
return null;
}
boolean reachedLimit() {
return remaining <= 0;
}
/**
* Returns the number of characters that have been read.
*
* Includes newline characters.
* Excludes any partial line data read before the limit is reached.
*/
long getCharsProcessed() {
return charsProcessed;
}
}
/**
* Does this policy effectively allow everything? (No
* disallows or timing (crawl-delay) directives?)
modules/src/test/java/org/archive/modules/net/RobotstxtTest.java
+20 -18
View File
@@ -20,6 +20,7 @@ package org.archive.modules.net;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.nio.ByteBuffer;
@@ -29,7 +30,7 @@ import org.archive.bdb.AutoKryo;
public class RobotstxtTest extends TestCase {
public void testParseRobots() throws IOException {
BufferedReader reader = new BufferedReader(new StringReader("BLAH"));
Reader reader = new StringReader("BLAH");
Robotstxt r = new Robotstxt(reader);
assertFalse(r.hasErrors);
assertEquals(0,r.getNamedUserAgents().size());
@@ -57,8 +58,7 @@ public class RobotstxtTest extends TestCase {
}
static Robotstxt sampleRobots1() throws IOException {
BufferedReader reader = new BufferedReader(
new StringReader(
Reader reader = new StringReader(
"User-agent: *\n" +
"Disallow: /cgi-bin/\n" +
"Disallow: /details/software\n" +
@@ -78,13 +78,12 @@ public class RobotstxtTest extends TestCase {
"Disallow: /\n" +
"Crawl-Delay: 20\n"+
"Allow: /images/\n"
));
);
return new Robotstxt(reader);
}
Robotstxt whitespaceFlawedRobots() throws IOException {
BufferedReader reader = new BufferedReader(
new StringReader(
Reader reader = new StringReader(
" User-agent: *\n" +
" Disallow: /cgi-bin/\n" +
" Disallow: /details/software\n" +
@@ -100,7 +99,7 @@ public class RobotstxtTest extends TestCase {
" Disallow: /\n" +
" Crawl-Delay: 20\n"+
" Allow: /images/\n"
));
);
return new Robotstxt(reader);
}
@@ -144,8 +143,7 @@ public class RobotstxtTest extends TestCase {
}
Robotstxt htmlMarkupRobots() throws IOException {
BufferedReader reader = new BufferedReader(
new StringReader(
Reader reader = new StringReader(
"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\"><HTML>\n"
+"<HEAD>\n"
+"<TITLE>/robots.txt</TITLE>\n"
@@ -157,7 +155,7 @@ public class RobotstxtTest extends TestCase {
+"\n"
+"</BODY>\n"
+"</HTML>\n"
));
);
return new Robotstxt(reader);
}
@@ -188,7 +186,7 @@ public class RobotstxtTest extends TestCase {
"Disallow:/service\n";
StringReader sr = new StringReader(TEST_ROBOTS_TXT);
Robotstxt rt = new Robotstxt(new BufferedReader(sr));
Robotstxt rt = new Robotstxt(sr);
{
RobotsDirectives da = rt.getDirectivesFor("a", false);
RobotsDirectives db = rt.getDirectivesFor("b", false);
@@ -216,7 +214,7 @@ public class RobotstxtTest extends TestCase {
+ "User-agent: a\n"
+ "Crawl-delay: 99\n";
StringReader sr = new StringReader(TEST_ROBOTS_TXT);
Robotstxt rt = new Robotstxt(new BufferedReader(sr));
Robotstxt rt = new Robotstxt(sr);
assertFalse(rt.getDirectivesFor("a").allows("/foo"));
@@ -231,14 +229,18 @@ public class RobotstxtTest extends TestCase {
public void testSizeLimit() throws IOException {
StringBuilder builder = new StringBuilder(
"User-agent: a\n" +
" Disallow: /\n" +
"User-Agent: b\n");
" Disallow: /\n" +
"User-Agent: b\nDisallow: /");
for (int i = 0; i < Robotstxt.MAX_SIZE; i++) {
builder.append(' ');
}
builder.append("Disallow: /\n");
Robotstxt rt = new Robotstxt(new BufferedReader(new StringReader(builder.toString())));
assertFalse("we should parse the first part", rt.getDirectivesFor("a").allows("/foo"));
assertTrue("but ignore anything after the size limit", rt.getDirectivesFor("b").allows("/foo"));
builder.append("\nUser-Agent: c\nDisallow: /\n");
Robotstxt rt = new Robotstxt(new StringReader(builder.toString()));
assertFalse("we should parse the first few lines",
rt.getDirectivesFor("a").allows("/foo"));
assertTrue("ignore the line that breaks the size limit",
rt.getDirectivesFor("b").allows("/foo"));
assertTrue("and also ignore any lines after the size limit",
rt.getDirectivesFor("c").allows("/foo"));
}
}