lee/heritrix3
1
0
Fork 0
mirror of https://github.com/internetarchive/heritrix3.git synced 2026-09-24 14:45:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #192 from internetarchive/robotstxt-size-limit

Browse Source 
Enforce robots.txt character limit per char not per line
This commit is contained in:
Noah Levitt
2017-11-01 17:20:30 -07:00
committed by GitHub
parent f0d1c0422b d6d7f1b162
commit deda35eedd
3 changed files with 69 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/src/main/java/org/archive/modules/net/CrawlServer.java
+1 -2
View File
@@ -174,10 +174,9 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden
InputStream contentBodyStream = null;
try {
BufferedReader reader;
contentBodyStream = curi.getRecorder().getContentReplayInputStream();
reader = new BufferedReader(new InputStreamReader(contentBodyStream));
InputStreamReader reader = new InputStreamReader(contentBodyStream);
robotstxt = new Robotstxt(reader);
validRobots = true;
} catch (IOException e) {
modules/src/main/java/org/archive/modules/net/Robotstxt.java
+32 -29
View File
@@ -18,15 +18,17 @@
*/
package org.archive.modules.net;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.Serializable;
import java.nio.CharBuffer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.archive.bdb.AutoKryo;
@@ -42,7 +44,8 @@ public class Robotstxt implements Serializable {
private static final Logger logger =
Logger.getLogger(Robotstxt.class.getName());
protected static final long MAX_SIZE = 500*1024;
protected static final int MAX_SIZE = 500*1024;
private static final Pattern LINE_SEPARATOR = Pattern.compile("\r\n|\r|\n");
// all user agents contained in this robots.txt
// in order of declaration
@@ -62,12 +65,16 @@ public class Robotstxt implements Serializable {
public Robotstxt() {
}
public Robotstxt(BufferedReader reader) throws IOException {
initializeFromReader(reader);
public Robotstxt(Reader reader) throws IOException {
try {
initializeFromReader(reader);
} finally {
IOUtils.closeQuietly(reader);
}
}
public Robotstxt(ReadSource customRobots) {
BufferedReader reader = new BufferedReader(customRobots.obtainReader());
Reader reader = customRobots.obtainReader();
try {
initializeFromReader(reader);
} catch (IOException e) {
@@ -79,32 +86,28 @@ public class Robotstxt implements Serializable {
}
}
protected void initializeFromReader(BufferedReader reader) throws IOException {
String read;
long charCount = 0;
protected void initializeFromReader(Reader reader) throws IOException {
CharBuffer buffer = CharBuffer.allocate(MAX_SIZE);
while (buffer.hasRemaining() && reader.read(buffer) >= 0) ;
buffer.flip();
String[] lines = LINE_SEPARATOR.split(buffer);
if (buffer.limit() == buffer.capacity()) {
int processed = buffer.capacity();
if (lines.length != 0) {
// discard the partial line at the end so we don't process a truncated path
int last = lines.length - 1;
processed -= lines[last].length();
lines[last] = "";
}
logger.warning("processed " + processed + " characters, ignoring the rest (see HER-1990)");
}
// current is the disallowed paths for the preceding User-Agent(s)
RobotsDirectives current = null;
while (reader != null) {
// we count characters instead of bytes because the byte count isn't easily available
if (charCount >= MAX_SIZE) {
logger.warning("processed " + charCount + " characters, ignoring the rest (see HER-1990)");
reader.close();
reader = null;
continue;
}
do {
read = reader.readLine();
if (read != null) {
charCount += read.length();
}
// Skip comments & blanks
} while (read != null && charCount < MAX_SIZE
&& ((read = read.trim()).startsWith("#") || read.length() == 0));
if (read == null) {
reader.close();
reader = null;
} else {
for (String read: lines) {
read = read.trim();
if (!read.isEmpty() && !read.startsWith("#")) {
// remove any html markup
read = read.replaceAll("<[^>]+>","");
int commentIndex = read.indexOf("#");
modules/src/test/java/org/archive/modules/net/RobotstxtTest.java
+36 -12
View File
@@ -20,6 +20,7 @@ package org.archive.modules.net;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.nio.ByteBuffer;
@@ -29,7 +30,7 @@ import org.archive.bdb.AutoKryo;
public class RobotstxtTest extends TestCase {
public void testParseRobots() throws IOException {
BufferedReader reader = new BufferedReader(new StringReader("BLAH"));
Reader reader = new StringReader("BLAH");
Robotstxt r = new Robotstxt(reader);
assertFalse(r.hasErrors);
assertEquals(0,r.getNamedUserAgents().size());
@@ -57,8 +58,7 @@ public class RobotstxtTest extends TestCase {
}
static Robotstxt sampleRobots1() throws IOException {
BufferedReader reader = new BufferedReader(
new StringReader(
Reader reader = new StringReader(
"User-agent: *\n" +
"Disallow: /cgi-bin/\n" +
"Disallow: /details/software\n" +
@@ -78,13 +78,12 @@ public class RobotstxtTest extends TestCase {
"Disallow: /\n" +
"Crawl-Delay: 20\n"+
"Allow: /images/\n"
));
);
return new Robotstxt(reader);
}
Robotstxt whitespaceFlawedRobots() throws IOException {
BufferedReader reader = new BufferedReader(
new StringReader(
Reader reader = new StringReader(
" User-agent: *\n" +
" Disallow: /cgi-bin/\n" +
" Disallow: /details/software\n" +
@@ -100,7 +99,7 @@ public class RobotstxtTest extends TestCase {
" Disallow: /\n" +
" Crawl-Delay: 20\n"+
" Allow: /images/\n"
));
);
return new Robotstxt(reader);
}
@@ -144,8 +143,7 @@ public class RobotstxtTest extends TestCase {
}
Robotstxt htmlMarkupRobots() throws IOException {
BufferedReader reader = new BufferedReader(
new StringReader(
Reader reader = new StringReader(
"<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\"><HTML>\n"
+"<HEAD>\n"
+"<TITLE>/robots.txt</TITLE>\n"
@@ -157,7 +155,7 @@ public class RobotstxtTest extends TestCase {
+"\n"
+"</BODY>\n"
+"</HTML>\n"
));
);
return new Robotstxt(reader);
}
@@ -188,7 +186,7 @@ public class RobotstxtTest extends TestCase {
"Disallow:/service\n";
StringReader sr = new StringReader(TEST_ROBOTS_TXT);
Robotstxt rt = new Robotstxt(new BufferedReader(sr));
Robotstxt rt = new Robotstxt(sr);
{
RobotsDirectives da = rt.getDirectivesFor("a", false);
RobotsDirectives db = rt.getDirectivesFor("b", false);
@@ -216,7 +214,7 @@ public class RobotstxtTest extends TestCase {
+ "User-agent: a\n"
+ "Crawl-delay: 99\n";
StringReader sr = new StringReader(TEST_ROBOTS_TXT);
Robotstxt rt = new Robotstxt(new BufferedReader(sr));
Robotstxt rt = new Robotstxt(sr);
assertFalse(rt.getDirectivesFor("a").allows("/foo"));
@@ -227,4 +225,30 @@ public class RobotstxtTest extends TestCase {
assertEquals(99f, rt.getDirectivesFor("a").getCrawlDelay());
}
public void testSizeLimit() throws IOException {
StringBuilder builder = new StringBuilder(
"User-agent: a\n" +
" Disallow: /\n" +
"User-Agent: b\nDisallow: /");
for (int i = 0; i < Robotstxt.MAX_SIZE; i++) {
builder.append(' ');
}
builder.append("\nUser-Agent: c\nDisallow: /\n");
Robotstxt rt = new Robotstxt(new StringReader(builder.toString()));
assertFalse("we should parse the first few lines",
rt.getDirectivesFor("a").allows("/foo"));
assertTrue("ignore the line that breaks the size limit",
rt.getDirectivesFor("b").allows("/foo"));
assertTrue("and also ignore any lines after the size limit",
rt.getDirectivesFor("c").allows("/foo"));
}
public void testAllBlankLines() throws IOException {
StringBuilder builder = new StringBuilder();
for (int i = 0; i < Robotstxt.MAX_SIZE; i++) {
builder.append('\n');
}
new Robotstxt(new StringReader(builder.toString()));
}
}