diff --git a/commons/src/main/java/org/archive/io/CrawlerJournal.java b/commons/src/main/java/org/archive/io/CrawlerJournal.java index ee595ad3..42b74756 100644 --- a/commons/src/main/java/org/archive/io/CrawlerJournal.java +++ b/commons/src/main/java/org/archive/io/CrawlerJournal.java @@ -1,43 +1,33 @@ -/* CrawlerJournal.java +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Created on Mar 6, 2007 + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Copyright (C) 2007 Internet Archive. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * http://www.apache.org/licenses/LICENSE-2.0 * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.io; import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; import it.unimi.dsi.mg4j.util.MutableString; -import java.io.BufferedInputStream; -import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.Writer; import java.util.List; -import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import org.archive.checkpointing.RecoverAction; @@ -56,25 +46,6 @@ public class CrawlerJournal { public static final String LOG_ERROR = "E "; /** prefix for timestamp lines */ public static final String LOG_TIMESTAMP = "T "; - - /** - * Get a BufferedReader on the crawler journal given - * - * TODO: move to a general utils class - * - * @param source File journal - * @return journal buffered reader. - * @throws IOException - */ - public static BufferedReader getBufferedReader(File source) throws IOException { - InputStream is = new BufferedInputStream(new FileInputStream(source)); - boolean isGzipped = source.getName().toLowerCase(). - endsWith(GZIP_SUFFIX); - if(isGzipped) { - is = new GZIPInputStream(is); - } - return new BufferedReader(new InputStreamReader(is)); - } /** * Stream on which we record frontier events. @@ -85,10 +56,6 @@ public class CrawlerJournal { protected long lines = 0; /** number of lines between timestamps */ protected int timestamp_interval = 0; // 0 means no timestamps - - - /** suffix to recognize gzipped files */ - public static final String GZIP_SUFFIX = ".gz"; /** * File we're writing journal to. diff --git a/commons/src/main/java/org/archive/util/IoUtils.java b/commons/src/main/java/org/archive/util/IoUtils.java index 625406f9..f6288708 100644 --- a/commons/src/main/java/org/archive/util/IoUtils.java +++ b/commons/src/main/java/org/archive/util/IoUtils.java @@ -23,6 +23,7 @@ import it.unimi.dsi.fastutil.io.FastBufferedOutputStream; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.BufferedReader; import java.io.Closeable; import java.io.EOFException; import java.io.File; @@ -30,6 +31,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.OutputStream; @@ -40,6 +42,7 @@ import java.util.Iterator; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.SerializationUtils; @@ -328,4 +331,25 @@ public class IoUtils { public static Object deserializeFromByteArray(byte[] in) { return SerializationUtils.deserialize(in); } + + /** suffix to recognize gzipped files */ + public static final String GZIP_SUFFIX = ".gz"; + /** + * Get a BufferedReader on the crawler journal given + * + * TODO: move to a general utils class + * + * @param source File journal + * @return journal buffered reader. + * @throws IOException + */ + public static BufferedReader getBufferedReader(File source) throws IOException { + InputStream is = new BufferedInputStream(new FileInputStream(source)); + boolean isGzipped = source.getName().toLowerCase(). + endsWith(GZIP_SUFFIX); + if(isGzipped) { + is = new GZIPInputStream(is); + } + return new BufferedReader(new InputStreamReader(is)); + } } diff --git a/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java index 7d4300be..09f85bcd 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java +++ b/engine/src/main/java/org/archive/crawler/frontier/FrontierJournal.java @@ -1,26 +1,20 @@ -/* RecoveryJournal +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * $Id$ + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Created on Jul 20, 2004 + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Copyright (C) 2004 Internet Archive. + * http://www.apache.org/licenses/LICENSE-2.0 * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.crawler.frontier; @@ -41,6 +35,7 @@ import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideRule; import org.archive.net.UURI; import org.archive.net.UURIFactory; +import org.archive.util.IoUtils; import org.json.JSONObject; /** @@ -62,6 +57,7 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable { public final static String F_ADD = "F+ "; public final static String F_EMIT = "Fe "; + public final static String F_INCLUDE = "Fi "; public final static String F_DISREGARD = "Fd "; public final static String F_RESCHEDULE = "Fr "; public final static String F_SUCCESS = "Fs "; @@ -111,6 +107,11 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable { writeLine(F_EMIT, curi.toString()); } + + public void included(CrawlURI curi) { + writeLine(F_INCLUDE, curi.toString()); + + } public void finishedFailure(CrawlURI curi) { writeLongUriLine(F_FAILURE,curi); @@ -194,7 +195,7 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable { DecideRule scope = (scopeIncludes) ? frontier.getScope() : null; FrontierJournal newJournal = frontier.getFrontierJournal(); - BufferedReader br = getBufferedReader(source); + BufferedReader br = IoUtils.getBufferedReader(source); String read; int lines = 0; try { @@ -273,7 +274,7 @@ public class FrontierJournal extends CrawlerJournal implements Checkpointable { try { // Scan log for all 'F+' lines: if not alreadyIncluded, schedule for // visitation - br = getBufferedReader(source); + br = IoUtils.getBufferedReader(source); try { while ((read = br.readLine())!=null) { qLines++; diff --git a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java index 9a683ae4..48c2815e 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java +++ b/engine/src/main/java/org/archive/crawler/frontier/precedence/PrecedenceLoader.java @@ -1,27 +1,25 @@ -/* PrecedenceLoader.java - * - * Created on Nov 27, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2007 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.crawler.frontier.precedence; +import static org.archive.modules.CoreAttributeConstants.A_PRECALC_PRECEDENCE; + import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; @@ -31,10 +29,8 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import static org.archive.modules.CoreAttributeConstants.*; - -import org.archive.io.CrawlerJournal; import org.archive.modules.recrawl.PersistProcessor; +import org.archive.util.IoUtils; import org.archive.util.bdbje.EnhancedEnvironment; import org.archive.util.iterator.LineReadingIterator; @@ -123,7 +119,7 @@ public class PrecedenceLoader { if(source.isFile()) { // scan log, writing to database - BufferedReader br = CrawlerJournal.getBufferedReader(source); + BufferedReader br = IoUtils.getBufferedReader(source); Iterator iter = new LineReadingIterator(br); while(iter.hasNext()) { String line = (String) iter.next(); diff --git a/engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java b/engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java index 357f182b..c28efdab 100644 --- a/engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java +++ b/engine/src/main/java/org/archive/crawler/util/RecoveryLogMapper.java @@ -1,27 +1,22 @@ -/* RecoveryLogMapper.java -* -* $Id$ -* -* Created on Mar 7, 2005 -* -* Copyright (C) 2005 Mike Schwartz. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /** * Parses a Heritrix recovery log file (recover.gz), and builds maps @@ -41,6 +36,7 @@ package org.archive.crawler.util; import org.archive.crawler.frontier.FrontierJournal; +import org.archive.util.IoUtils; import java.io.File; import java.io.LineNumberReader; @@ -116,8 +112,8 @@ public class RecoveryLogMapper { protected void load(String recoverLogFileName) throws java.io.FileNotFoundException, java.io.IOException, SeedUrlNotFoundException { - LineNumberReader reader = new LineNumberReader(FrontierJournal. - getBufferedReader(new File(recoverLogFileName))); + LineNumberReader reader = new LineNumberReader( + IoUtils.getBufferedReader(new File(recoverLogFileName))); String curLine = null; while ((curLine = reader.readLine()) != null) { if (curLine.length() == 0 diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java index 351693fc..f79b89c9 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java @@ -1,25 +1,22 @@ -/* PersistProcessor.java - * - * Created on Feb 17, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2007 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + package org.archive.modules.recrawl; import java.io.BufferedReader; @@ -37,10 +34,10 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.SerializationUtils; import org.archive.bdb.BdbModule; -import org.archive.io.CrawlerJournal; import org.archive.modules.Processor; import org.archive.modules.ProcessorURI; import org.archive.util.ArchiveUtils; +import org.archive.util.IoUtils; import org.archive.util.SURT; import org.archive.util.bdbje.EnhancedEnvironment; import org.archive.util.iterator.LineReadingIterator; @@ -180,7 +177,7 @@ public abstract class PersistProcessor extends Processor { if(source.isFile()) { // scan log, writing to database - BufferedReader br = CrawlerJournal.getBufferedReader(source); + BufferedReader br = IoUtils.getBufferedReader(source); Iterator iter = new LineReadingIterator(br); while(iter.hasNext()) { String line = (String) iter.next(); @@ -249,7 +246,7 @@ public abstract class PersistProcessor extends Processor { if(source.isFile()) { // scan log, writing to database - BufferedReader br = CrawlerJournal.getBufferedReader(source); + BufferedReader br = IoUtils.getBufferedReader(source); Iterator iter = new LineReadingIterator(br); while(iter.hasNext()) { String line = (String) iter.next();