diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index 11aaf477..c223df19 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -174,10 +174,9 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats, Iden InputStream contentBodyStream = null; try { - BufferedReader reader; contentBodyStream = curi.getRecorder().getContentReplayInputStream(); - reader = new BufferedReader(new InputStreamReader(contentBodyStream)); + InputStreamReader reader = new InputStreamReader(contentBodyStream); robotstxt = new Robotstxt(reader); validRobots = true; } catch (IOException e) { diff --git a/modules/src/main/java/org/archive/modules/net/Robotstxt.java b/modules/src/main/java/org/archive/modules/net/Robotstxt.java index b61f5822..c11bd34f 100644 --- a/modules/src/main/java/org/archive/modules/net/Robotstxt.java +++ b/modules/src/main/java/org/archive/modules/net/Robotstxt.java @@ -18,15 +18,17 @@ */ package org.archive.modules.net; -import java.io.BufferedReader; import java.io.IOException; +import java.io.Reader; import java.io.Serializable; +import java.nio.CharBuffer; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.archive.bdb.AutoKryo; @@ -42,7 +44,8 @@ public class Robotstxt implements Serializable { private static final Logger logger = Logger.getLogger(Robotstxt.class.getName()); - protected static final long MAX_SIZE = 500*1024; + protected static final int MAX_SIZE = 500*1024; + private static final Pattern LINE_SEPARATOR = Pattern.compile("\r\n|\r|\n"); // all user agents contained in this robots.txt // in order of declaration @@ -62,12 +65,16 @@ public class Robotstxt implements Serializable { public Robotstxt() { } - public Robotstxt(BufferedReader reader) throws IOException { - initializeFromReader(reader); + public Robotstxt(Reader reader) throws IOException { + try { + initializeFromReader(reader); + } finally { + IOUtils.closeQuietly(reader); + } } public Robotstxt(ReadSource customRobots) { - BufferedReader reader = new BufferedReader(customRobots.obtainReader()); + Reader reader = customRobots.obtainReader(); try { initializeFromReader(reader); } catch (IOException e) { @@ -79,32 +86,28 @@ public class Robotstxt implements Serializable { } } - protected void initializeFromReader(BufferedReader reader) throws IOException { - String read; - long charCount = 0; + protected void initializeFromReader(Reader reader) throws IOException { + CharBuffer buffer = CharBuffer.allocate(MAX_SIZE); + while (buffer.hasRemaining() && reader.read(buffer) >= 0) ; + buffer.flip(); + + String[] lines = LINE_SEPARATOR.split(buffer); + if (buffer.limit() == buffer.capacity()) { + int processed = buffer.capacity(); + if (lines.length != 0) { + // discard the partial line at the end so we don't process a truncated path + int last = lines.length - 1; + processed -= lines[last].length(); + lines[last] = ""; + } + logger.warning("processed " + processed + " characters, ignoring the rest (see HER-1990)"); + } + // current is the disallowed paths for the preceding User-Agent(s) RobotsDirectives current = null; - while (reader != null) { - // we count characters instead of bytes because the byte count isn't easily available - if (charCount >= MAX_SIZE) { - logger.warning("processed " + charCount + " characters, ignoring the rest (see HER-1990)"); - reader.close(); - reader = null; - continue; - } - - do { - read = reader.readLine(); - if (read != null) { - charCount += read.length(); - } - // Skip comments & blanks - } while (read != null && charCount < MAX_SIZE - && ((read = read.trim()).startsWith("#") || read.length() == 0)); - if (read == null) { - reader.close(); - reader = null; - } else { + for (String read: lines) { + read = read.trim(); + if (!read.isEmpty() && !read.startsWith("#")) { // remove any html markup read = read.replaceAll("<[^>]+>",""); int commentIndex = read.indexOf("#"); diff --git a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java index 1f529077..ec102294 100644 --- a/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java +++ b/modules/src/test/java/org/archive/modules/net/RobotstxtTest.java @@ -20,6 +20,7 @@ package org.archive.modules.net; import java.io.BufferedReader; import java.io.IOException; +import java.io.Reader; import java.io.StringReader; import java.nio.ByteBuffer; @@ -29,7 +30,7 @@ import org.archive.bdb.AutoKryo; public class RobotstxtTest extends TestCase { public void testParseRobots() throws IOException { - BufferedReader reader = new BufferedReader(new StringReader("BLAH")); + Reader reader = new StringReader("BLAH"); Robotstxt r = new Robotstxt(reader); assertFalse(r.hasErrors); assertEquals(0,r.getNamedUserAgents().size()); @@ -57,8 +58,7 @@ public class RobotstxtTest extends TestCase { } static Robotstxt sampleRobots1() throws IOException { - BufferedReader reader = new BufferedReader( - new StringReader( + Reader reader = new StringReader( "User-agent: *\n" + "Disallow: /cgi-bin/\n" + "Disallow: /details/software\n" + @@ -78,13 +78,12 @@ public class RobotstxtTest extends TestCase { "Disallow: /\n" + "Crawl-Delay: 20\n"+ "Allow: /images/\n" - )); + ); return new Robotstxt(reader); } Robotstxt whitespaceFlawedRobots() throws IOException { - BufferedReader reader = new BufferedReader( - new StringReader( + Reader reader = new StringReader( " User-agent: *\n" + " Disallow: /cgi-bin/\n" + " Disallow: /details/software\n" + @@ -100,7 +99,7 @@ public class RobotstxtTest extends TestCase { " Disallow: /\n" + " Crawl-Delay: 20\n"+ " Allow: /images/\n" - )); + ); return new Robotstxt(reader); } @@ -144,8 +143,7 @@ public class RobotstxtTest extends TestCase { } Robotstxt htmlMarkupRobots() throws IOException { - BufferedReader reader = new BufferedReader( - new StringReader( + Reader reader = new StringReader( "\n" +"\n" +"/robots.txt\n" @@ -157,7 +155,7 @@ public class RobotstxtTest extends TestCase { +"\n" +"\n" +"\n" - )); + ); return new Robotstxt(reader); } @@ -188,7 +186,7 @@ public class RobotstxtTest extends TestCase { "Disallow:/service\n"; StringReader sr = new StringReader(TEST_ROBOTS_TXT); - Robotstxt rt = new Robotstxt(new BufferedReader(sr)); + Robotstxt rt = new Robotstxt(sr); { RobotsDirectives da = rt.getDirectivesFor("a", false); RobotsDirectives db = rt.getDirectivesFor("b", false); @@ -216,7 +214,7 @@ public class RobotstxtTest extends TestCase { + "User-agent: a\n" + "Crawl-delay: 99\n"; StringReader sr = new StringReader(TEST_ROBOTS_TXT); - Robotstxt rt = new Robotstxt(new BufferedReader(sr)); + Robotstxt rt = new Robotstxt(sr); assertFalse(rt.getDirectivesFor("a").allows("/foo")); @@ -227,4 +225,30 @@ public class RobotstxtTest extends TestCase { assertEquals(99f, rt.getDirectivesFor("a").getCrawlDelay()); } + + public void testSizeLimit() throws IOException { + StringBuilder builder = new StringBuilder( + "User-agent: a\n" + + " Disallow: /\n" + + "User-Agent: b\nDisallow: /"); + for (int i = 0; i < Robotstxt.MAX_SIZE; i++) { + builder.append(' '); + } + builder.append("\nUser-Agent: c\nDisallow: /\n"); + Robotstxt rt = new Robotstxt(new StringReader(builder.toString())); + assertFalse("we should parse the first few lines", + rt.getDirectivesFor("a").allows("/foo")); + assertTrue("ignore the line that breaks the size limit", + rt.getDirectivesFor("b").allows("/foo")); + assertTrue("and also ignore any lines after the size limit", + rt.getDirectivesFor("c").allows("/foo")); + } + + public void testAllBlankLines() throws IOException { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < Robotstxt.MAX_SIZE; i++) { + builder.append('\n'); + } + new Robotstxt(new StringReader(builder.toString())); + } }