Move getBufferedReader() -> IoUtils

* CrawlerJournal.java, IoUtils.java
    move getBufferedReader to IoUtils
* FrontierJournal.java
    add new 'included' tag for log lines ("Fi")
* (others)
    update for relocated method

engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java

@@ -1,26 +1,20 @@
-/* RecoveryJournal
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
  *
- * $Id$
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
  *
- * Created on Jul 20, 2004
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
  *
- * Copyright (C) 2004 Internet Archive.
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- *
- * Heritrix is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- *
- * Heritrix is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser Public License for more details.
- *
- * You should have received a copy of the GNU Lesser Public License
- * along with Heritrix; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
 package org.archive.crawler.frontier;
 
@@ -41,6 +35,7 @@ import org.archive.modules.CrawlURI;
 import org.archive.modules.deciderules.DecideRule;
 import org.archive.net.UURI;
 import org.archive.net.UURIFactory;
+import org.archive.util.IoUtils;
 import org.json.JSONObject;
 
 /**
@@ -62,6 +57,7 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
 
     public final static String F_ADD = "F+ ";
     public final static String F_EMIT = "Fe ";
+    public final static String F_INCLUDE = "Fi ";
     public final static String F_DISREGARD = "Fd ";
     public final static String F_RESCHEDULE = "Fr ";
     public final static String F_SUCCESS = "Fs ";
@@ -111,6 +107,11 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
         writeLine(F_EMIT, curi.toString());
 
     }
+    
+    public void included(CrawlURI curi) {
+        writeLine(F_INCLUDE, curi.toString());
+
+    }
 
     public void finishedFailure(CrawlURI curi) {
         writeLongUriLine(F_FAILURE,curi);
@@ -194,7 +195,7 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
         DecideRule scope = (scopeIncludes) ? frontier.getScope() : null;
         FrontierJournal newJournal = frontier.getFrontierJournal();
         
-        BufferedReader br = getBufferedReader(source);
+        BufferedReader br = IoUtils.getBufferedReader(source);
         String read;
         int lines = 0; 
         try {
@@ -273,7 +274,7 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable {
         try {
             // Scan log for all 'F+' lines: if not alreadyIncluded, schedule for
             // visitation
-            br = getBufferedReader(source);
+            br = IoUtils.getBufferedReader(source);
             try {
                 while ((read = br.readLine())!=null) {
                     qLines++;

engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java

@@ -1,27 +1,25 @@
-/* PrecedenceLoader.java
- * 
- * Created on Nov 27, 2005
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
  *
- * Copyright (C) 2007 Internet Archive.
- * 
- * This file is part of the Heritrix web crawler (crawler.archive.org).
- * 
- * Heritrix is free software; you can redistribute it and/or modify
- * it under the terms of the GNU Lesser Public License as published by
- * the Free Software Foundation; either version 2.1 of the License, or
- * any later version.
- * 
- * Heritrix is distributed in the hope that it will be useful, 
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser Public License for more details.
- * 
- * You should have received a copy of the GNU Lesser Public License
- * along with Heritrix; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
 package org.archive.crawler.frontier.precedence;
 
+import static org.archive.modules.CoreAttributeConstants.A_PRECALC_PRECEDENCE;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -31,10 +29,8 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 
-import static org.archive.modules.CoreAttributeConstants.*;
-
-import org.archive.io.CrawlerJournal;
 import org.archive.modules.recrawl.PersistProcessor;
+import org.archive.util.IoUtils;
 import org.archive.util.bdbje.EnhancedEnvironment;
 import org.archive.util.iterator.LineReadingIterator;
 
@@ -123,7 +119,7 @@ public class PrecedenceLoader {
         
         if(source.isFile()) {
             // scan log, writing to database
-            BufferedReader br = CrawlerJournal.getBufferedReader(source);
+            BufferedReader br = IoUtils.getBufferedReader(source);
             Iterator iter = new LineReadingIterator(br);
             while(iter.hasNext()) {
                 String line = (String) iter.next(); 

engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java

@@ -1,27 +1,22 @@
-/* RecoveryLogMapper.java
-*
-* $Id$
-*
-* Created on Mar 7, 2005
-*
-* Copyright (C) 2005 Mike Schwartz.
-*
-* This file is part of the Heritrix web crawler (crawler.archive.org).
-*
-* Heritrix is free software; you can redistribute it and/or modify
-* it under the terms of the GNU Lesser Public License as published by
-* the Free Software Foundation; either version 2.1 of the License, or
-* any later version.
-*
-* Heritrix is distributed in the hope that it will be useful,
-* but WITHOUT ANY WARRANTY; without even the implied warranty of
-* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-* GNU Lesser Public License for more details.
-*
-* You should have received a copy of the GNU Lesser Public License
-* along with Heritrix; if not, write to the Free Software
-* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-*/
+/*
+ *  This file is part of the Heritrix web crawler (crawler.archive.org).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
 
 /**
  * Parses a Heritrix recovery log file (recover.gz), and builds maps
@@ -41,6 +36,7 @@
 package org.archive.crawler.util;
 
 import org.archive.crawler.frontier.FrontierJournal;
+import org.archive.util.IoUtils;
 
 import java.io.File;
 import java.io.LineNumberReader;
@@ -116,8 +112,8 @@ public class RecoveryLogMapper {
     protected void load(String recoverLogFileName)
     throws java.io.FileNotFoundException, java.io.IOException,
             SeedUrlNotFoundException {
-        LineNumberReader reader = new LineNumberReader(FrontierJournal.
-            getBufferedReader(new File(recoverLogFileName)));
+        LineNumberReader reader = new LineNumberReader(
+                IoUtils.getBufferedReader(new File(recoverLogFileName)));
         String curLine = null;
         while ((curLine = reader.readLine()) != null) {
             if (curLine.length() == 0