mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-08-07 07:11:19 +00:00
copy 'springy' branch to heritrix3
This commit is contained in:
gojomo
@@ -0,0 +1,69 @@
|
||||
/* CandidateURITest.java
|
||||
*
|
||||
* $Id$
|
||||
*
|
||||
* Created Jun 23, 2005
|
||||
*
|
||||
* Copyright (C) 2005 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.datamodel;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.archive.net.UURIFactory;
|
||||
|
||||
/**
|
||||
* Test CandidateURI serialization.
|
||||
* @author stack
|
||||
*/
|
||||
public class CandidateURITest extends TestCase {
|
||||
public void testSerialization()
|
||||
throws IOException, ClassNotFoundException {
|
||||
doOneSerialization("http://www.archive.org/");
|
||||
doOneSerialization("http://www.archive.org/a?" +
|
||||
"sch=%2E%2F%3Faction%3Dsearch");
|
||||
}
|
||||
|
||||
private void doOneSerialization(final String urlStr)
|
||||
throws IOException, ClassNotFoundException {
|
||||
CrawlURI cauri =
|
||||
new CrawlURI(UURIFactory.getInstance(urlStr));
|
||||
cauri = serialize(cauri);
|
||||
assertEquals(urlStr + " doesn't serialize", urlStr,
|
||||
cauri.getUURI().toString());
|
||||
}
|
||||
|
||||
private CrawlURI serialize(CrawlURI cauri)
|
||||
throws IOException, ClassNotFoundException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ObjectOutputStream oos = new ObjectOutputStream(baos);
|
||||
oos.writeObject(cauri);
|
||||
oos.flush();
|
||||
oos.close();
|
||||
ByteArrayInputStream bais =
|
||||
new ByteArrayInputStream(baos.toByteArray());
|
||||
return (CrawlURI)(new ObjectInputStream(bais)).readObject();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/* CrawlURITest
|
||||
*
|
||||
* Created on Jul 26, 2004
|
||||
*
|
||||
* Copyright (C) 2004 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.datamodel;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.TmpDirTestCase;
|
||||
|
||||
/**
|
||||
* @author stack
|
||||
* @version $Revision$, $Date$
|
||||
*/
|
||||
public class CrawlURITest extends TmpDirTestCase {
|
||||
|
||||
CrawlURI seed = null;
|
||||
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
final String url = "http://www.dh.gov.uk/Home/fs/en";
|
||||
this.seed = new CrawlURI(UURIFactory.getInstance(url));
|
||||
this.seed.setSchedulingDirective(SchedulingConstants.MEDIUM);
|
||||
this.seed.setSeed(true);
|
||||
// Force caching of string.
|
||||
this.seed.toString();
|
||||
// TODO: should this via really be itself?
|
||||
this.seed.setVia(UURIFactory.getInstance(url));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test serialization/deserialization works.
|
||||
*
|
||||
* @throws IOException
|
||||
* @throws ClassNotFoundException
|
||||
*/
|
||||
final public void testSerialization()
|
||||
throws IOException, ClassNotFoundException {
|
||||
File serialize = new File(getTmpDir(),
|
||||
this.getClass().getName() + ".serialize");
|
||||
try {
|
||||
FileOutputStream fos = new FileOutputStream(serialize);
|
||||
ObjectOutputStream oos = new ObjectOutputStream(fos);
|
||||
oos.writeObject(this.seed);
|
||||
oos.reset();
|
||||
oos.writeObject(this.seed);
|
||||
oos.reset();
|
||||
oos.writeObject(this.seed);
|
||||
oos.close();
|
||||
// Read in the object.
|
||||
FileInputStream fis = new FileInputStream(serialize);
|
||||
ObjectInputStream ois = new ObjectInputStream(fis);
|
||||
CrawlURI deserializedCuri = (CrawlURI)ois.readObject();
|
||||
deserializedCuri = (CrawlURI)ois.readObject();
|
||||
deserializedCuri = (CrawlURI)ois.readObject();
|
||||
assertEquals("Deserialized not equal to original",
|
||||
this.seed.toString(), deserializedCuri.toString());
|
||||
String host = this.seed.getUURI().getHost();
|
||||
assertTrue("Deserialized host not null",
|
||||
host != null && host.length() >= 0);
|
||||
} finally {
|
||||
serialize.delete();
|
||||
}
|
||||
}
|
||||
|
||||
public void testCandidateURIWithLoadedAList()
|
||||
throws URIException {
|
||||
UURI uuri = UURIFactory.getInstance("http://www.archive.org");
|
||||
CrawlURI c = new CrawlURI(uuri);
|
||||
c.setSeed(true);
|
||||
c.getData().put("key", "value");
|
||||
CrawlURI curi = new CrawlURI(c, 0);
|
||||
assertTrue("Didn't find AList item",
|
||||
curi.getData().get("key").equals("value"));
|
||||
}
|
||||
|
||||
// TODO: move to QueueAssignmentPolicies
|
||||
// public void testCalculateClassKey() throws URIException {
|
||||
// final String uri = "http://mprsrv.agri.gov.cn";
|
||||
// CrawlURI curi = new CrawlURI(UURIFactory.getInstance(uri));
|
||||
// String key = curi.getClassKey();
|
||||
// assertTrue("Key1 is bad " + key,
|
||||
// key.equals(curi.getUURI().getAuthorityMinusUserinfo()));
|
||||
// final String baduri = "ftp://pfbuser:pfbuser@mprsrv.agri.gov.cn/clzreceive/";
|
||||
// curi = new CrawlURI(UURIFactory.getInstance(baduri));
|
||||
// key = curi.getClassKey();
|
||||
// assertTrue("Key2 is bad " + key,
|
||||
// key.equals(curi.getUURI().getAuthorityMinusUserinfo()));
|
||||
// }
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.framework;
|
||||
|
||||
import static org.archive.util.TmpDirTestCase.DEFAULT_TEST_TMP_DIR;
|
||||
import static org.archive.util.TmpDirTestCase.TEST_TMP_SYSTEM_PROPERTY_NAME;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
|
||||
import org.archive.bdb.BdbModule;
|
||||
import org.archive.modules.net.BdbServerCache;
|
||||
import org.archive.spring.ConfigPath;
|
||||
import org.archive.state.ModuleTestBase;
|
||||
import org.archive.util.IoUtils;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class CrawlControllerTest extends ModuleTestBase {
|
||||
|
||||
// TODO TESTME
|
||||
|
||||
public static CrawlController makeTempCrawlController() throws Exception {
|
||||
String tmpPath = System.getProperty(TEST_TMP_SYSTEM_PROPERTY_NAME);
|
||||
if (tmpPath == null) {
|
||||
tmpPath = DEFAULT_TEST_TMP_DIR;
|
||||
}
|
||||
File tmp = new File(tmpPath);
|
||||
if (!tmp.exists()) {
|
||||
tmp.mkdirs();
|
||||
}
|
||||
|
||||
FileWriter fileWriter = null;
|
||||
try {
|
||||
fileWriter = new FileWriter(new File(tmp, "seeds.txt"));
|
||||
fileWriter.write("http://www.pandemoniummovie.com");
|
||||
fileWriter.close();
|
||||
} finally {
|
||||
IoUtils.close(fileWriter);
|
||||
}
|
||||
|
||||
File state = new File(tmp, "state");
|
||||
state.mkdirs();
|
||||
|
||||
File checkpoints = new File(tmp, "checkpoints");
|
||||
checkpoints.mkdirs();
|
||||
|
||||
BdbModule bdb = new BdbModule();
|
||||
bdb.setDir(new ConfigPath("test",state.getAbsolutePath()));
|
||||
// def.set(bdb, BdbModule.DIR, state.getAbsolutePath());
|
||||
bdb.start();
|
||||
|
||||
String cp = checkpoints.getAbsolutePath();
|
||||
|
||||
CrawlController controller = new CrawlController();
|
||||
controller.setServerCache(new BdbServerCache());
|
||||
controller.setCheckpointsDir(new ConfigPath("test",cp));
|
||||
controller.start();
|
||||
return controller;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void verifySerialization(Object first, byte[] firstBytes,
|
||||
Object second, byte[] secondBytes) throws Exception {
|
||||
// TODO TESTME
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* CrawlerProcessor.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.framework;
|
||||
|
||||
|
||||
import org.archive.modules.ProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link CrawlerProcessor}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public abstract class CrawlerProcessorTestBase extends ProcessorTestBase {
|
||||
|
||||
|
||||
protected CrawlController controller;
|
||||
|
||||
|
||||
protected void setUp() throws Exception {
|
||||
controller = CrawlControllerTest.makeTempCrawlController();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void verifySerialization(Object first, byte[] firstBytes,
|
||||
Object second, byte[] secondBytes) throws Exception {
|
||||
|
||||
}
|
||||
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* AdaptiveRevisitFrontierTest.java
|
||||
*
|
||||
* Created on Feb 5, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.frontier;
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* @author pjack
|
||||
*
|
||||
*/
|
||||
public class BdbFrontierTest extends CrawlerProcessorTestBase {
|
||||
|
||||
@Override
|
||||
public void testSerialization() {
|
||||
// FIXME
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void verifySerialization(Object first, byte[] firstBytes,
|
||||
Object second, byte[] secondBytes) throws Exception {
|
||||
}
|
||||
|
||||
|
||||
// TODO TESTME
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
/* BdbMultipleWorkQueuesTest
|
||||
*
|
||||
* $Id$
|
||||
*
|
||||
* Created on Jul 21, 2005
|
||||
*
|
||||
* Copyright (C) 2005 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.frontier;
|
||||
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.crawler.datamodel.CrawlURI;
|
||||
import org.archive.crawler.datamodel.SchedulingConstants;
|
||||
import org.archive.net.UURIFactory;
|
||||
|
||||
import com.sleepycat.je.tree.Key;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
/**
|
||||
* Unit tests for BdbMultipleWorkQueues functionality.
|
||||
*
|
||||
* @author gojomo
|
||||
*/
|
||||
public class BdbMultipleWorkQueuesTest extends TestCase {
|
||||
private static Logger logger =
|
||||
Logger.getLogger(BdbMultipleWorkQueuesTest.class.getName());
|
||||
|
||||
|
||||
/**
|
||||
* Basic sanity checks for calculateInsertKey() -- ensure ordinal, cost,
|
||||
* and schedulingDirective have the intended effects, for ordinal values
|
||||
* up through 1/4th of the maximum (about 2^61).
|
||||
*
|
||||
* @throws URIException
|
||||
*/
|
||||
public void testCalculateInsertKey() throws URIException {
|
||||
while(Thread.interrupted()) {
|
||||
logger.warning("stray interrupt cleared");
|
||||
}
|
||||
|
||||
for (long ordinalOrigin = 1; ordinalOrigin < Long.MAX_VALUE / 4; ordinalOrigin <<= 1) {
|
||||
CrawlURI cauri1 =
|
||||
new CrawlURI(UURIFactory.getInstance("http://archive.org/foo"));
|
||||
CrawlURI curi1 = new CrawlURI(cauri1, ordinalOrigin);
|
||||
curi1.setClassKey("foo");
|
||||
byte[] key1 =
|
||||
BdbMultipleWorkQueues.calculateInsertKey(curi1).getData();
|
||||
CrawlURI cauri2 =
|
||||
new CrawlURI(UURIFactory.getInstance("http://archive.org/bar"));
|
||||
CrawlURI curi2 = new CrawlURI(cauri2, ordinalOrigin + 1);
|
||||
curi2.setClassKey("foo");
|
||||
byte[] key2 =
|
||||
BdbMultipleWorkQueues.calculateInsertKey(curi2).getData();
|
||||
CrawlURI cauri3 =
|
||||
new CrawlURI(UURIFactory.getInstance("http://archive.org/baz"));
|
||||
CrawlURI curi3 = new CrawlURI(cauri3, ordinalOrigin + 2);
|
||||
curi3.setClassKey("foo");
|
||||
curi3.setSchedulingDirective(SchedulingConstants.HIGH);
|
||||
byte[] key3 =
|
||||
BdbMultipleWorkQueues.calculateInsertKey(curi3).getData();
|
||||
CrawlURI cauri4 =
|
||||
new CrawlURI(UURIFactory.getInstance("http://archive.org/zle"));
|
||||
CrawlURI curi4 = new CrawlURI(cauri4, ordinalOrigin + 3);
|
||||
curi4.setClassKey("foo");
|
||||
curi4.setPrecedence(2);
|
||||
byte[] key4 =
|
||||
BdbMultipleWorkQueues.calculateInsertKey(curi4).getData();
|
||||
CrawlURI cauri5 =
|
||||
new CrawlURI(UURIFactory.getInstance("http://archive.org/gru"));
|
||||
CrawlURI curi5 = new CrawlURI(cauri5, ordinalOrigin + 4);
|
||||
curi5.setClassKey("foo");
|
||||
curi5.setPrecedence(1);
|
||||
byte[] key5 =
|
||||
BdbMultipleWorkQueues.calculateInsertKey(curi5).getData();
|
||||
// ensure that key1 (with lower ordinal) sorts before key2 (higher
|
||||
// ordinal)
|
||||
assertTrue("lower ordinal sorting first (" + ordinalOrigin + ")",
|
||||
Key.compareKeys(key1, key2, null) < 0);
|
||||
// ensure that key3 (with HIGH scheduling) sorts before key2 (even
|
||||
// though
|
||||
// it has lower ordinal)
|
||||
assertTrue("lower directive sorting first (" + ordinalOrigin + ")",
|
||||
Key.compareKeys(key3, key2, null) < 0);
|
||||
// ensure that key5 (with lower cost) sorts before key4 (even though
|
||||
// key4 has lower ordinal and same default NORMAL scheduling directive)
|
||||
assertTrue("lower cost sorting first (" + ordinalOrigin + ")", Key
|
||||
.compareKeys(key5, key4, null) < 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (C) 2008 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SimpleQueuePrecedencePolicyTest.java
|
||||
*
|
||||
* Created on May 5, 2008
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.frontier;
|
||||
|
||||
import org.archive.state.ModuleTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Tests for BucketQueueAssignmentPolicy
|
||||
*/
|
||||
public class BucketQueueAssignmentPolicyTest extends ModuleTestBase {
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
/* RecoveryJournalTest
|
||||
*
|
||||
* Created on Apr 18, 2005
|
||||
*
|
||||
* Copyright (C) 2005 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.frontier;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.util.TmpDirTestCase;
|
||||
|
||||
/**
|
||||
* @author stack
|
||||
* @version $Date$, $Revision$
|
||||
*/
|
||||
public class FrontierJournalTest extends TmpDirTestCase {
|
||||
private FrontierJournal rj;
|
||||
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
this.rj = new FrontierJournal(this.getTmpDir().getAbsolutePath(),
|
||||
this.getClass().getName());
|
||||
}
|
||||
|
||||
protected void tearDown() throws Exception {
|
||||
super.tearDown();
|
||||
if (this.rj != null) {
|
||||
this.rj.close();
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String [] args) {
|
||||
junit.textui.TestRunner.run(FrontierJournalTest.class);
|
||||
}
|
||||
|
||||
public void testAdded() throws URIException {
|
||||
/*
|
||||
CandidateURI c = new CandidateURI(UURIFactory.
|
||||
getInstance("http://www.archive.org"), "LLLLL",
|
||||
UURIFactory.getInstance("http://archive.org"),
|
||||
"L");
|
||||
this.rj.added(new CrawlURI(c, 0));
|
||||
this.rj.added(new CrawlURI(c, 1));
|
||||
this.rj.added(new CrawlURI(c, 2));
|
||||
*/
|
||||
}
|
||||
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SimpleQueuePrecedencePolicyTest.java
|
||||
*
|
||||
* Created on Nov 17, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.frontier.precedence;
|
||||
|
||||
import org.archive.state.ModuleTestBase;
|
||||
|
||||
/**
|
||||
* Tests for BaseQueuePrecedencePolicy
|
||||
*/
|
||||
public class BaseQueuePrecedencePolicyTest extends ModuleTestBase {
|
||||
|
||||
//TODO add tests
|
||||
|
||||
}
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SimpleQueuePrecedencePolicyTest.java
|
||||
*
|
||||
* Created on Nov 17, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.frontier.precedence;
|
||||
|
||||
import org.archive.state.ModuleTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Tests for BaseUriPrecedencePolicy
|
||||
*/
|
||||
public class BaseUriPrecedencePolicyTest extends ModuleTestBase {
|
||||
|
||||
//TODO add tests
|
||||
|
||||
}
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SimpleQueuePrecedencePolicyTest.java
|
||||
*
|
||||
* Created on Nov 17, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.frontier.precedence;
|
||||
|
||||
import org.archive.state.ModuleTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Tests for CostUriPrecedencePolicy
|
||||
*/
|
||||
public class CostUriPrecedencePolicyTest extends ModuleTestBase {
|
||||
|
||||
//TODO add tests
|
||||
|
||||
}
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SuccessCountsQueuePrecedencePolicyTest.java
|
||||
*
|
||||
* Created on Nov 17, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.frontier.precedence;
|
||||
|
||||
import org.archive.state.ModuleTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Tests for HighestUriQueuePrecedencePolicy
|
||||
*/
|
||||
public class HighestUriQueuePrecedencePolicyTest extends ModuleTestBase {
|
||||
|
||||
//TODO add tests
|
||||
|
||||
}
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SimpleQueuePrecedencePolicyTest.java
|
||||
*
|
||||
* Created on Nov 17, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.frontier.precedence;
|
||||
|
||||
import org.archive.state.ModuleTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Tests for HopsUriPrecedencePolicy
|
||||
*/
|
||||
public class HopsUriPrecedencePolicyTest extends ModuleTestBase {
|
||||
|
||||
}
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SuccessCountsQueuePrecedencePolicyTest.java
|
||||
*
|
||||
* Created on Nov 17, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.frontier.precedence;
|
||||
|
||||
import org.archive.state.ModuleTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Tests for SuccessCountsQueuePrecedencePolicy
|
||||
*/
|
||||
public class SuccessCountsQueuePrecedencePolicyTest extends ModuleTestBase {
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* CrawlStateUpdater.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.postprocessor;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link CrawlStateUpdater}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class CrawlStateUpdaterTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* FrontierScheduler.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.postprocessor;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link FrontierScheduler}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class FrontierSchedulerTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* LinksScoper.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.postprocessor;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link LinksScoper}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class LinksScoperTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* LowDiskPauseProcessor.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.postprocessor;
|
||||
|
||||
|
||||
import org.archive.modules.ProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link LowDiskPauseProcessor}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class LowDiskPauseProcessorTest extends ProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SupplementaryLinksScoper.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.postprocessor;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link SupplementaryLinksScoper}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class SupplementaryLinksScoperTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* PreconditionEnforcer.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.prefetch;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link PreconditionEnforcer}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class PreconditionEnforcerTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* Preselector.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.prefetch;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link Preselector}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class PreselectorTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* QuotaEnforcer.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.prefetch;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link QuotaEnforcer}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class QuotaEnforcerTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* RuntimeLimitEnforcer.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.prefetch;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link RuntimeLimitEnforcer}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class RuntimeLimitEnforcerTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* HashCrawlMapper.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.processor;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link HashCrawlMapper}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class HashCrawlMapperTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* LexicalCrawlMapper.java
|
||||
*
|
||||
* Created on Jan 31, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.processor;
|
||||
|
||||
|
||||
import org.archive.crawler.framework.CrawlerProcessorTestBase;
|
||||
|
||||
|
||||
/**
|
||||
* Unit test for {@link LexicalCrawlMapper}.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class LexicalCrawlMapperTest extends CrawlerProcessorTestBase {
|
||||
|
||||
// TODO TESTME!
|
||||
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
/* BackgroundImageExtractionSelfTest
|
||||
*
|
||||
* Created on Jan 29, 2004
|
||||
*
|
||||
* Copyright (C) 2004 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
/**
|
||||
* Test the crawler can find background images in pages.
|
||||
*
|
||||
* @author stack
|
||||
* @version $Id$
|
||||
*/
|
||||
public class BackgroundImageExtractionSelfTestCase
|
||||
extends SelfTestBase
|
||||
{
|
||||
|
||||
|
||||
/**
|
||||
* Files to find as a set.
|
||||
*/
|
||||
final private static Set<String> EXPECTED = Collections.unmodifiableSet(
|
||||
new HashSet<String>(Arrays.asList(new String[] {
|
||||
"index.html", "example-background-image.jpeg", "robots.txt"
|
||||
})));
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
Set<String> files = filesInArcs();
|
||||
assertTrue(EXPECTED.equals(files));
|
||||
}
|
||||
|
||||
// TODO TESTME
|
||||
|
||||
|
||||
// /**
|
||||
// * The name of the background image the crawler is supposed to find.
|
||||
// */
|
||||
// private static final String IMAGE_NAME = "example-background-image.jpeg";
|
||||
//
|
||||
// private static final String JPEG = "image/jpeg";
|
||||
//
|
||||
//
|
||||
// /**
|
||||
// * Read ARC file for the background image the file that contained it.
|
||||
// *
|
||||
// * Look that there is only one instance of the background image in the
|
||||
// * ARC and that it is of the same size as the image in the webapp dir.
|
||||
// */
|
||||
// public void testBackgroundImageExtraction()
|
||||
// {
|
||||
// String relativePath = getTestName() + '/' + IMAGE_NAME;
|
||||
// String url = getSelftestURLWithTrailingSlash() + relativePath;
|
||||
// File image = new File(getHtdocs(), relativePath);
|
||||
// assertTrue("Image exists", image.exists());
|
||||
// List [] metaDatas = getMetaDatas();
|
||||
// boolean found = false;
|
||||
// ARCRecordMetaData metaData = null;
|
||||
// for (int mi = 0; mi < metaDatas.length; mi++) {
|
||||
// List list = metaDatas[mi];
|
||||
// for (final Iterator i = list.iterator(); i.hasNext();) {
|
||||
// metaData = (ARCRecordMetaData) i.next();
|
||||
// if (metaData.getUrl().equals(url)
|
||||
// && metaData.getMimetype().equalsIgnoreCase(JPEG)) {
|
||||
// if (!found) {
|
||||
// found = true;
|
||||
// } else {
|
||||
// fail("Found a 2nd instance of " + url);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/* BadURIsStopPageParsingSelfTest
|
||||
*
|
||||
* Created on Mar 10, 2004
|
||||
*
|
||||
* Copyright (C) 2004 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Selftest for figuring problems parsing URIs in a page.
|
||||
*
|
||||
* @author stack
|
||||
* @see <a
|
||||
* href="https://sourceforge.net/tracker/?func=detail&aid=788219&group_id=73833&atid=539099">[ 788219 ]
|
||||
* URI Syntax Errors stop page parsing.</a>
|
||||
* @version $Revision$, $Date$
|
||||
*/
|
||||
public class BadURIsStopPageParsingSelfTest extends SelfTestBase
|
||||
{
|
||||
|
||||
/**
|
||||
* Files to find as a set.
|
||||
*/
|
||||
final private static Set<String> EXPECTED = Collections.unmodifiableSet(
|
||||
new HashSet<String>(Arrays.asList(new String[] {
|
||||
"index.html", "goodone.html", "goodthree.html", "one.html",
|
||||
"two.html", "three.html", "robots.txt", "goodtwo.html",
|
||||
"cata;pgs-new.html", "www.loc.gov/rr/european/egw/polishex.html"
|
||||
})));
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
Set<String> files = filesInArcs();
|
||||
assertTrue(EXPECTED.equals(files));
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void verifyLogFileEmpty(String logFileName) {
|
||||
if (logFileName.equals("uri-errors.log")) {
|
||||
File logsDir = getLogsDir();
|
||||
File log = new File(logsDir, logFileName);
|
||||
if (log.length() == 0) {
|
||||
throw new IllegalStateException("Log " + logFileName +
|
||||
" is empty, expected URI failure.");
|
||||
}
|
||||
return;
|
||||
}
|
||||
super.verifyLogFileEmpty(logFileName);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/* CharsetSelfTest
|
||||
*
|
||||
* Created on Mar 10, 2004
|
||||
*
|
||||
* Copyright (C) 2004 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Simple test to ensure we can extract links from multibyte pages.
|
||||
*
|
||||
* That is, can we regex over a multibyte stream.
|
||||
*
|
||||
* @author stack
|
||||
* @version $Revision$, $Date$
|
||||
*/
|
||||
public class CharsetSelfTest extends SelfTestBase
|
||||
{
|
||||
final private static Set<String> EXPECTED = Collections.unmodifiableSet(
|
||||
new HashSet<String>(Arrays.asList(new String[] {
|
||||
"index.html", "link.html", "robots.txt"
|
||||
})));
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
Set<String> files = filesInArcs();
|
||||
assertTrue(EXPECTED.equals(files));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
/* $Id$
|
||||
*
|
||||
* Created Aug 15, 2006
|
||||
*
|
||||
* Copyright (C) 2006 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.archive.crawler.framework.CrawlJob;
|
||||
import org.mortbay.jetty.Server;
|
||||
import org.mortbay.jetty.bio.SocketConnector;
|
||||
import org.mortbay.jetty.servlet.ServletHandler;
|
||||
import org.mortbay.jetty.servlet.ServletHolder;
|
||||
|
||||
|
||||
/**
|
||||
* Assumes checkpoint was run during the SelfTest.
|
||||
* @author stack
|
||||
* @version $Date$ $Version$
|
||||
*/
|
||||
public class CheckpointSelfTest extends SelfTestBase {
|
||||
|
||||
final private static String HOST = "127.0.0.1";
|
||||
|
||||
final private static int MIN_PORT = 7000;
|
||||
|
||||
final private static int MAX_PORT = 7010;
|
||||
|
||||
final private static int MAX_HOPS = 1;
|
||||
|
||||
|
||||
private Server[] servers;
|
||||
|
||||
|
||||
public CheckpointSelfTest() {
|
||||
}
|
||||
|
||||
protected String getSeedsString() {
|
||||
String seedsString = "";
|
||||
for(int p = MIN_PORT; p <= MAX_PORT; p++) {
|
||||
seedsString += "http://127.0.0.1:"+p+"/random\\\n";
|
||||
}
|
||||
return seedsString;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void stopHttpServer() {
|
||||
boolean fail = false;
|
||||
for (int i = 0; i < servers.length; i++) try {
|
||||
servers[i].stop();
|
||||
} catch (Exception e) {
|
||||
fail = true;
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
if (fail) {
|
||||
throw new AssertionError();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startHttpServer() throws Exception {
|
||||
this.servers = new Server[MAX_PORT - MIN_PORT];
|
||||
for (int i = 0; i < servers.length; i++) {
|
||||
servers[i] = makeHttpServer(i + MIN_PORT);
|
||||
servers[i].start();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private Server makeHttpServer(int port) throws Exception {
|
||||
Server server = new Server();
|
||||
SocketConnector sc = new SocketConnector();
|
||||
sc.setHost(HOST);
|
||||
sc.setPort(port);
|
||||
server.addConnector(sc);
|
||||
ServletHandler servletHandler = new ServletHandler();
|
||||
server.setHandler(servletHandler);
|
||||
|
||||
RandomServlet random = new RandomServlet();
|
||||
random.setHost(HOST);
|
||||
random.setMinPort(MIN_PORT);
|
||||
random.setMaxPort(MAX_PORT);
|
||||
random.setMaxHops(MAX_HOPS);
|
||||
random.setPathRoot("random");
|
||||
|
||||
ServletHolder holder = new ServletHolder(random);
|
||||
servletHandler.addServletWithMapping(holder, "/random/*");
|
||||
server.start();
|
||||
return server;
|
||||
}
|
||||
|
||||
|
||||
// @Override
|
||||
protected void waitForCrawlFinish() throws Exception {
|
||||
|
||||
Thread.sleep(2000);
|
||||
CrawlJob cj = heritrix.getEngine().getJob("selftest-job");
|
||||
|
||||
// TODO: pause, checkpoint, launch new job, yadda yadda
|
||||
|
||||
// for now, just kill & wait
|
||||
cj.terminate();
|
||||
super.waitForCrawlFinish();
|
||||
|
||||
// invokeAndWait("basic", "requestCrawlPause", CrawlStatus.PAUSED);
|
||||
// invokeAndWait("basic", "requestCrawlCheckpoint", CrawlStatus.PAUSED);
|
||||
// invokeAndWait("basic", "requestCrawlStop", CrawlStatus.FINISHED);
|
||||
// waitFor("org.archive.crawler:*,name=basic,type=org.archive.crawler.framework.CrawlController", false);
|
||||
// stopHeritrix();
|
||||
// Set<ObjectName> set = dumpMBeanServer();
|
||||
// if (!set.isEmpty()) {
|
||||
// throw new Exception("Mbeans lived on after stopHeritrix: " + set);
|
||||
// }
|
||||
// this.heritrixThread = new HeritrixThread(new String[] {
|
||||
// "-j", getCrawlDir().getAbsolutePath() + "/jobs", "-n"
|
||||
// });
|
||||
// this.heritrixThread.start();
|
||||
//
|
||||
// ObjectName cjm = getEngine();
|
||||
// String[] checkpoints = (String[])server.invoke(
|
||||
// cjm,
|
||||
// "listCheckpoints",
|
||||
// new Object[] { "completed-basic" },
|
||||
// new String[] { "java.lang.String" });
|
||||
//
|
||||
// assertEquals(1, checkpoints.length);
|
||||
// File recoverLoc = new File(getCompletedJobDir().getParentFile(), "recovered");
|
||||
// FileUtils.deleteDir(recoverLoc);
|
||||
// String[] oldPath = new String[] { getCompletedJobDir().getAbsolutePath() };
|
||||
// String[] newPath = new String[] { recoverLoc.getAbsolutePath() };
|
||||
// server.invoke(
|
||||
// cjm,
|
||||
// "recoverCheckpoint",
|
||||
// new Object[] {
|
||||
// "completed-basic",
|
||||
// "active-recovered",
|
||||
// checkpoints[0],
|
||||
// oldPath,
|
||||
// newPath
|
||||
// },
|
||||
// new String[] {
|
||||
// String.class.getName(),
|
||||
// String.class.getName(),
|
||||
// String.class.getName(),
|
||||
// "java.lang.String[]",
|
||||
// "java.lang.String[]"
|
||||
// });
|
||||
// ObjectName cc = getCrawlController("recovered");
|
||||
// waitFor(cc);
|
||||
// invokeAndWait("recovered", "requestCrawlResume", CrawlStatus.FINISHED);
|
||||
//
|
||||
// server.invoke(
|
||||
// cjm,
|
||||
// "closeSheetManagerStub",
|
||||
// new Object[] { "completed-basic" },
|
||||
// new String[] { "java.lang.String" });
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
protected void verifyCommon() throws IOException {
|
||||
// checkpointing rotated the logs so default behavior won't work here
|
||||
// FIXME: Make this work :)
|
||||
|
||||
}
|
||||
|
||||
|
||||
protected void verify() throws Exception {
|
||||
// FIXME: Complete test.
|
||||
// assertTrue("neither feature nor test yet implemented",false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Repeat core testSomething 100 times. Rename to JUNit convention
|
||||
* to enable.
|
||||
*/
|
||||
public void xestSomething100() {
|
||||
for(int i = 0; i < 100; i++) {
|
||||
try {
|
||||
testSomething();
|
||||
} catch (Exception e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
// @Override
|
||||
// public void testSomething() throws Exception {
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.mortbay.jetty.Handler;
|
||||
import org.mortbay.jetty.Server;
|
||||
import org.mortbay.jetty.bio.SocketConnector;
|
||||
import org.mortbay.jetty.handler.DefaultHandler;
|
||||
import org.mortbay.jetty.handler.HandlerList;
|
||||
import org.mortbay.jetty.handler.ResourceHandler;
|
||||
import org.mortbay.jetty.servlet.ServletHandler;
|
||||
import org.mortbay.jetty.servlet.ServletHolder;
|
||||
|
||||
/**
|
||||
* Test form-based authentication
|
||||
*
|
||||
* @contributor stack
|
||||
* @contributor gojomo
|
||||
*/
|
||||
public class FormAuthSelfTest
|
||||
extends SelfTestBase
|
||||
{
|
||||
/**
|
||||
* Files to find as a list.
|
||||
*/
|
||||
final private static Set<String> EXPECTED = Collections.unmodifiableSet(
|
||||
new HashSet<String>(Arrays.asList(new String[] {
|
||||
"login/login.html", "success.html", "robots.txt"
|
||||
})));
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
Set<String> found = this.filesInArcs();
|
||||
assertEquals("wrong files in ARCs",EXPECTED,found);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startHttpServer() throws Exception {
|
||||
Server server = new Server();
|
||||
|
||||
SocketConnector sc = new SocketConnector();
|
||||
sc.setHost("localhost");
|
||||
sc.setPort(7777);
|
||||
server.addConnector(sc);
|
||||
ResourceHandler rhandler = new ResourceHandler();
|
||||
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
|
||||
|
||||
ServletHandler servletHandler = new ServletHandler();
|
||||
|
||||
HandlerList handlers = new HandlerList();
|
||||
handlers.setHandlers(new Handler[] {
|
||||
rhandler,
|
||||
servletHandler,
|
||||
new DefaultHandler() });
|
||||
server.setHandler(handlers);
|
||||
|
||||
ServletHolder holder = new ServletHolder(new FormAuthServlet());
|
||||
servletHandler.addServletWithMapping(holder, "/login/*");
|
||||
|
||||
this.httpServer = server;
|
||||
this.httpServer.start();
|
||||
}
|
||||
|
||||
protected String getSeedsString() {
|
||||
return "http://127.0.0.1:7777/login/login.html";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String changeGlobalConfig(String config) {
|
||||
String newCredStore =
|
||||
"<bean id=\"credentialStore\" class=\"org.archive.modules.credential.CredentialStore\">\n" +
|
||||
" <property name=\"credentials\">\n" +
|
||||
" <map>\n" +
|
||||
" <entry key=\"test2\">\n" +
|
||||
" <bean class=\"org.archive.modules.credential.HtmlFormCredential\">\n" +
|
||||
" <property name=\"domain\" value=\"127.0.0.1:7777\"/>\n" +
|
||||
" <property name=\"loginUri\" value=\"http://127.0.0.1:7777/login/login.html\"/>\n" +
|
||||
" <property name=\"formItems\">\n" +
|
||||
" <map>\n" +
|
||||
" <entry key=\"username\" value=\"Mr. Happy Pants\"/>\n" +
|
||||
" <entry key=\"password\" value=\"xyzzy\"/>\n" +
|
||||
" </map>\n" +
|
||||
" </property>\n" +
|
||||
" </bean>\n" +
|
||||
" </entry>\n" +
|
||||
" </map>\n" +
|
||||
" </property>\n" +
|
||||
"</bean>";
|
||||
config = config.replaceFirst(
|
||||
"(?s)<bean id=\"credentialStore\".*?</bean>",
|
||||
newCredStore);
|
||||
return super.changeGlobalConfig(config);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* AuthServlet.java
|
||||
*
|
||||
* Created on Feb 23, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
|
||||
/**
|
||||
* @author pjack
|
||||
*/
|
||||
public class FormAuthServlet extends HttpServlet {
|
||||
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
|
||||
@Override
|
||||
protected void doPost(HttpServletRequest req, HttpServletResponse resp)
|
||||
throws ServletException, IOException {
|
||||
String username = req.getParameter("username");
|
||||
String password = req.getParameter("password");
|
||||
if (username.equals("Mr. Happy Pants") && password.equals("xyzzy")) {
|
||||
resp.sendRedirect("/success.html");
|
||||
} else {
|
||||
resp.sendRedirect("/failure.html");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/* FramesSelfTest
|
||||
*
|
||||
* Created on Feb 6, 2004
|
||||
*
|
||||
* Copyright (C) 2004 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
/**
|
||||
* Test crawler can parse pages w/ frames in them.
|
||||
*
|
||||
* @author stack
|
||||
* @version $Id$
|
||||
*/
|
||||
public class FramesSelfTestCase extends SelfTestBase
|
||||
{
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Files to find as a set.
|
||||
*/
|
||||
final private static Set<String> EXPECTED = Collections.unmodifiableSet(
|
||||
new HashSet<String>(Arrays.asList(new String[] {
|
||||
"index.html", "topframe.html", "leftframe.html", "noframe.html",
|
||||
"rightframe.html", "robots.txt"
|
||||
})));
|
||||
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
Set<String> files = filesInArcs();
|
||||
assertTrue(EXPECTED.equals(files));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
import org.mortbay.jetty.Handler;
|
||||
import org.mortbay.jetty.Server;
|
||||
import org.mortbay.jetty.bio.SocketConnector;
|
||||
import org.mortbay.jetty.handler.DefaultHandler;
|
||||
import org.mortbay.jetty.handler.HandlerList;
|
||||
import org.mortbay.jetty.handler.ResourceHandler;
|
||||
import org.mortbay.jetty.security.Constraint;
|
||||
import org.mortbay.jetty.security.ConstraintMapping;
|
||||
import org.mortbay.jetty.security.HashUserRealm;
|
||||
import org.mortbay.jetty.security.SecurityHandler;
|
||||
import org.mortbay.jetty.servlet.ServletHandler;
|
||||
|
||||
/**
|
||||
* Test HTTP basic authentication
|
||||
*
|
||||
* @contributor stack
|
||||
* @contributor gojomo
|
||||
*/
|
||||
public class HttpAuthSelfTest
|
||||
extends SelfTestBase
|
||||
{
|
||||
/**
|
||||
* Files to find as a list.
|
||||
*/
|
||||
final private static Set<String> EXPECTED = Collections.unmodifiableSet(
|
||||
new HashSet<String>(Arrays.asList(new String[] {
|
||||
"index.html", "link1.html", "link2.html", "link3.html",
|
||||
"basic/index.html", "basic/link1.html", "basic/link2.html", "basic/link3.html",
|
||||
"robots.txt"
|
||||
})));
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
Set<String> found = this.filesInArcs();
|
||||
assertEquals("wrong files in ARCs",EXPECTED,found);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startHttpServer() throws Exception {
|
||||
Server server = new Server();
|
||||
|
||||
Constraint constraint = new Constraint();
|
||||
constraint.setName(Constraint.__BASIC_AUTH);;
|
||||
constraint.setRoles(new String[]{"user","admin","moderator"});
|
||||
constraint.setAuthenticate(true);
|
||||
|
||||
ConstraintMapping cm = new ConstraintMapping();
|
||||
cm.setConstraint(constraint);
|
||||
cm.setPathSpec("/basic/*");
|
||||
|
||||
HashUserRealm realm = new HashUserRealm();
|
||||
realm.setName("Hyrule");
|
||||
realm.put("Mr. Happy Pants", "xyzzy");
|
||||
realm.addUserToRole("Mr. Happy Pants", "user");
|
||||
|
||||
SecurityHandler securityHandler = new SecurityHandler();
|
||||
securityHandler.setUserRealm(realm);
|
||||
securityHandler.setConstraintMappings(new ConstraintMapping[]{cm});
|
||||
|
||||
SocketConnector sc = new SocketConnector();
|
||||
sc.setHost("localhost");
|
||||
sc.setPort(7777);
|
||||
server.addConnector(sc);
|
||||
ResourceHandler rhandler = new ResourceHandler();
|
||||
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
|
||||
|
||||
ServletHandler servletHandler = new ServletHandler();
|
||||
|
||||
HandlerList handlers = new HandlerList();
|
||||
handlers.setHandlers(new Handler[] {
|
||||
securityHandler,
|
||||
rhandler,
|
||||
servletHandler,
|
||||
new DefaultHandler() });
|
||||
server.setHandler(handlers);
|
||||
|
||||
this.httpServer = server;
|
||||
this.httpServer.start();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String changeGlobalConfig(String config) {
|
||||
String newCredStore =
|
||||
"<bean id=\"credentialStore\" class=\"org.archive.modules.credential.CredentialStore\">\n" +
|
||||
" <property name=\"credentials\">\n" +
|
||||
" <map>\n" +
|
||||
" <entry key=\"test\">\n" +
|
||||
" <bean class=\"org.archive.modules.credential.Rfc2617Credential\">\n" +
|
||||
" <property name=\"domain\" value=\"127.0.0.1:7777\"/>\n" +
|
||||
" <property name=\"realm\" value=\"Hyrule\"/>\n" +
|
||||
" <property name=\"login\" value=\"Mr. Happy Pants\"/>\n" +
|
||||
" <property name=\"password\" value=\"xyzzy\"/>\n" +
|
||||
" </bean>\n" +
|
||||
" </entry>\n" +
|
||||
" </map>\n" +
|
||||
" </property>\n" +
|
||||
"</bean>";
|
||||
config = config.replaceFirst(
|
||||
"(?s)<bean id=\"credentialStore\".*?</bean>",
|
||||
newCredStore);
|
||||
return super.changeGlobalConfig(config);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.archive.crawler.datamodel.CrawlURI;
|
||||
import org.archive.modules.Processor;
|
||||
import org.archive.modules.ProcessorURI;
|
||||
|
||||
|
||||
/**
|
||||
* An example analysis module that prioritizes outlinks of URIs that contain
|
||||
* a certain keyword over the outlinks of URIs that do not.
|
||||
*
|
||||
* <p>This is just a proof-of-concept; it isn't appropriate for actual
|
||||
* production crawls, and so it lives with the test code. This module has
|
||||
* the following limitations:
|
||||
*
|
||||
* <ol>
|
||||
* <li>It doesn't parse HTML content; so trying to match a keyword of "body"
|
||||
* would match.</li>
|
||||
* <li>It doesn't do any language analysis (eg, "political" if "politics" is
|
||||
* the specified keyword).</li>
|
||||
* <li>It can't match more than one keyword.</li>
|
||||
* <li>It doesn't consider the number of times the keyword appears.</li>
|
||||
* </ol>
|
||||
*
|
||||
* And so on. However, this module does provide a simple example of how to
|
||||
* modify precedence values of a URI's links based on that URI's content.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class KeyWordProcessor extends Processor {
|
||||
private static final long serialVersionUID = 1L;
|
||||
/**
|
||||
* Regular expression used to detect the presence of a keyword.
|
||||
*/
|
||||
Pattern pattern = Pattern.compile("\\bkeyword\\b");
|
||||
public Pattern getPattern() {
|
||||
return this.pattern;
|
||||
}
|
||||
public void setPattern(Pattern pattern) {
|
||||
this.pattern = pattern;
|
||||
}
|
||||
|
||||
/**
|
||||
* Precedence value to assign to discovered links of URIs that match
|
||||
* the pattern.
|
||||
*/
|
||||
int foundPrecedence = 1;
|
||||
public int getFoundPrecedence() {
|
||||
return this.foundPrecedence;
|
||||
}
|
||||
public void setFoundPrecedence(int prec) {
|
||||
this.foundPrecedence = prec;
|
||||
}
|
||||
|
||||
/**
|
||||
* Precedence value to assign to discovered links of URIs that do not
|
||||
* match the pattern.
|
||||
*/
|
||||
int notFoundPrecedence = 10;
|
||||
public int getNotFoundPrecedence() {
|
||||
return this.notFoundPrecedence;
|
||||
}
|
||||
public void setNotFoundPrecedence(int prec) {
|
||||
this.notFoundPrecedence = prec;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void innerProcess(ProcessorURI uri) throws InterruptedException {
|
||||
CrawlURI curi = (CrawlURI)uri;
|
||||
try {
|
||||
CharSequence seq = uri.getRecorder().getReplayCharSequence();
|
||||
int precedence;
|
||||
if (getPattern().matcher(seq).find()) {
|
||||
precedence = getFoundPrecedence();
|
||||
} else {
|
||||
precedence = getNotFoundPrecedence();
|
||||
}
|
||||
for (CrawlURI c: curi.getOutCandidates()) {
|
||||
c.setPrecedence(precedence);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean shouldProcess(ProcessorURI uri) {
|
||||
if (!uri.getContentType().equals("text/html")) {
|
||||
return false;
|
||||
}
|
||||
return uri instanceof CrawlURI;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import org.archive.crawler.datamodel.CrawlURI;
|
||||
import org.archive.crawler.frontier.precedence.BaseUriPrecedencePolicy;
|
||||
|
||||
/**
|
||||
* Testing policy which uses a precedence inside the CrawlURI (presumably
|
||||
* put there earlier by KeyWordProcessor).
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class KeyWordUriPrecedencePolicy extends BaseUriPrecedencePolicy {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
@Override
|
||||
protected int calculatePrecedence(CrawlURI curi) {
|
||||
if (curi.getPrecedence() > 0) {
|
||||
return curi.getPrecedence();
|
||||
}
|
||||
return super.calculatePrecedence(curi);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/* MaxLinkHopsSelfTest
|
||||
*
|
||||
* Created on Feb 17, 2004
|
||||
*
|
||||
* Copyright (C) 2004 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*/
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Test the max-link-hops setting.
|
||||
*
|
||||
* @author stack
|
||||
* @version $Id$
|
||||
*/
|
||||
public class MaxLinkHopsSelfTest
|
||||
extends SelfTestBase
|
||||
{
|
||||
final private static Set<String> EXPECTED = Collections.unmodifiableSet(
|
||||
new HashSet<String>(Arrays.asList(new String[] {
|
||||
"index.html", "1.html", "2.html", "3.html", "robots.txt"
|
||||
})));
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
Set<String> files = filesInArcs();
|
||||
assertEquals("ARC contents not as expected",EXPECTED,files);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String changeGlobalConfig(String config) {
|
||||
String replacement =
|
||||
"<bean class=\"org.archive.modules.deciderules.TooManyHopsDecideRule\">\n" +
|
||||
" <property name=\"maxHops\" value=\"3\"/>\n" +
|
||||
" </bean>";
|
||||
String retVal = config.replaceFirst(
|
||||
"(?s)<bean class=\"org.archive.modules.deciderules.TooManyHopsDecideRule\".*?</bean>",
|
||||
replacement);
|
||||
return super.changeGlobalConfig(retVal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
|
||||
import org.archive.crawler.frontier.precedence.BaseUriPrecedencePolicy;
|
||||
import org.archive.util.IoUtils;
|
||||
|
||||
/**
|
||||
* Tests that operators can create precedence groups for URIs, and that URIs
|
||||
* in one group are crawled before URIs in another group per operator preference.
|
||||
*
|
||||
* <p>The embedded Jetty HTTP server for this test provides the following
|
||||
* document tree:
|
||||
*
|
||||
* <ul>
|
||||
* <li>seed.html</li>
|
||||
* <li>one/</li>
|
||||
* <ul>
|
||||
* <li>a.html</li>
|
||||
* <li>b.html</li>
|
||||
* <li>c.html</li>
|
||||
* </ul>
|
||||
* <li>five/</li>
|
||||
* <ul>
|
||||
* <li>a.html</li>
|
||||
* <li>b.html</li>
|
||||
* <li>c.html</li>
|
||||
* </ul>
|
||||
* <li>ten/</li>
|
||||
* <ul>
|
||||
* <li>a.html</li>
|
||||
* <li>b.html</li>
|
||||
* <li>c.html</li>
|
||||
* </ul>
|
||||
* </ul>
|
||||
*
|
||||
* (See the <code>engine/testdata/selftest/Precedence1SelfTest</code>
|
||||
* directory to view these files.) The <code>seed.html</code> file contains
|
||||
* links to <code>five/a.html</code>, <code>ten/a.html</code>, and
|
||||
* <code>one/a.html</code>, in that order. The <code>a.html</code> files link
|
||||
* to to the <code>b.html</code> files, and the <code>b.html</code> link to
|
||||
* the <code>c.html</code> files, which have no out links.
|
||||
*
|
||||
* <p>Ordinarily Heritrix would crawl these in (roughly) the order the links
|
||||
* are discovered:
|
||||
*
|
||||
* <ol>
|
||||
* <li>seed.html</li>
|
||||
* <li>five/a.html</li>
|
||||
* <li>ten/a.html</li>
|
||||
* <li>one/a.html</li>
|
||||
* <li>five/b.html</li>
|
||||
* <li>ten/b.html</li>
|
||||
* <li>one/b.html</li>
|
||||
* <li>five/c.html</li>
|
||||
* <li>ten/c.html</li>
|
||||
* <li>one/c.html</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>However, the crawl configuration for this test uses a
|
||||
* {@link BaseUriPrecedencePolicy} instead of the default
|
||||
* {@link org.archive.crawler.frontier.policy.CostUriPrecedencePolicy}. The
|
||||
* <code>BasePrecedencePolicy</code> is configured so that all URIs have a
|
||||
* precedence value of 5 unless otherwise specified.
|
||||
*
|
||||
* <p>There is a sheet named <code>HiPri</code> that overrides the
|
||||
* <code>base-precedence</code> to be 1 instead of 5; thus URIs associated
|
||||
* with the HiPri sheet should be crawled before other URIs.
|
||||
* Similarly, there is a sheet named <code>LoPri</code> that overrides
|
||||
* <code>base-precedence</code> to be 10 instead of 5. URLs associated with
|
||||
* LoPri should be crawled after other URLs.
|
||||
*
|
||||
* <p>The <code>one/</code> directory is associated with the HiPri sheet, and
|
||||
* the <code>ten/</code> directory is associated with the LoPri sheet. This
|
||||
* creates three "groups" of URIs: one, five and ten. All of the URIs in
|
||||
* group "one" should be crawled before any of the URIs in group "five" are
|
||||
* crawled. Similarly, all of the URIs in group "five" should be crawled before
|
||||
* any of the URIs in group "ten".
|
||||
*
|
||||
* <p>So the final order in which URLs should be crawled in this test is:
|
||||
*
|
||||
* <ol>
|
||||
* <li>seed.html</li>
|
||||
* <li>one/a.html</li>
|
||||
* <li>one/b.html</li>
|
||||
* <li>one/c.html</li>
|
||||
* <li>five/a.html</li>
|
||||
* <li>five/b.html</li>
|
||||
* <li>five/c.html</li>
|
||||
* <li>ten/a.html</li>
|
||||
* <li>ten/b.html</li>
|
||||
* <li>ten/c.html</li>
|
||||
* </ol>
|
||||
*
|
||||
* This tests ensures that the documents were crawled in the correct order.
|
||||
*
|
||||
* <p>Although this test uses the directory structure of the URIs to group the URIs
|
||||
* into precedence groups, because the test executes on just one machine.
|
||||
* But the same basic configuration could be used to group URIs by any SURT
|
||||
* prefix -- by host or by domain, even by top-level domain. So an operator
|
||||
* could associate HiPri with all .gov sites to ensure that all .gov URIs
|
||||
* are crawled before any non-.gov URIs.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class Precedence1SelfTest extends SelfTestBase {
|
||||
|
||||
|
||||
/**
|
||||
* Expected results of the crawl.
|
||||
*/
|
||||
final private static String EXPECTED =
|
||||
"http://127.0.0.1:7777/robots.txt\n" +
|
||||
"http://127.0.0.1:7777/seed.html\n" +
|
||||
"http://127.0.0.1:7777/one/a.html\n" +
|
||||
"http://127.0.0.1:7777/one/b.html\n" +
|
||||
"http://127.0.0.1:7777/one/c.html\n" +
|
||||
"http://127.0.0.1:7777/five/a.html\n" +
|
||||
"http://127.0.0.1:7777/five/b.html\n" +
|
||||
"http://127.0.0.1:7777/five/c.html\n" +
|
||||
"http://127.0.0.1:7777/ten/a.html\n" +
|
||||
"http://127.0.0.1:7777/ten/b.html\n" +
|
||||
"http://127.0.0.1:7777/ten/c.html\n";
|
||||
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
File crawlLog = new File(getLogsDir(), "crawl.log");
|
||||
BufferedReader br = null;
|
||||
String crawled = "";
|
||||
try {
|
||||
br = new BufferedReader(new FileReader(crawlLog));
|
||||
for (String s = br.readLine(); s != null; s = br.readLine()) {
|
||||
s = s.substring(42);
|
||||
int i = s.indexOf(' ');
|
||||
s = s.substring(0, i);
|
||||
crawled = crawled + s + "\n";
|
||||
}
|
||||
} finally {
|
||||
IoUtils.close(br);
|
||||
}
|
||||
|
||||
assertEquals(EXPECTED, crawled);
|
||||
}
|
||||
|
||||
protected String getSeedsString() {
|
||||
return "http://127.0.0.1:7777/seed.html";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String changeGlobalConfig(String config) {
|
||||
// add a uriPrecedencePolicy with overlayable values, IF replaced
|
||||
// string not already gone (as if by subclass)
|
||||
String uriPrecedencePolicy =
|
||||
"<property name='uriPrecedencePolicy'>\n" +
|
||||
" <bean class='org.archive.crawler.frontier.precedence.BaseUriPrecedencePolicy'>\n" +
|
||||
" <property name='basePrecedence' value='5'/>\n" +
|
||||
" </bean>" +
|
||||
"</property>";
|
||||
config = config.replace("<!--@@FRONTIER_PROPERTIES@@-->", uriPrecedencePolicy);
|
||||
|
||||
config = configureSheets(config);
|
||||
return super.changeGlobalConfig(config);
|
||||
}
|
||||
|
||||
protected String configureSheets(String config) {
|
||||
// add sheets which overlay alternate precedence values for some URIs
|
||||
String sheets =
|
||||
"<bean id='loPri' class='org.archive.crawler.spring.SheetForSurtPrefixes'>\n" +
|
||||
" <property name='surtPrefixes'>\n" +
|
||||
" <list>\n" +
|
||||
" <value>http://(127.0.0.1:7777)/ten</value>\n" +
|
||||
" </list>\n" +
|
||||
" </property>\n" +
|
||||
" <property name='map'>\n" +
|
||||
" <map>\n" +
|
||||
" <entry key='frontier.uriPrecedencePolicy.basePrecedence' value='10'/>\n" +
|
||||
" </map>\n" +
|
||||
" </property>\n" +
|
||||
"</bean>\n" +
|
||||
"<bean id='hiPri' class='org.archive.crawler.spring.SheetForSurtPrefixes'>\n" +
|
||||
" <property name='surtPrefixes'>\n" +
|
||||
" <list>\n" +
|
||||
" <value>http://(127.0.0.1:7777)/one</value>\n" +
|
||||
" </list>\n" +
|
||||
" </property>\n" +
|
||||
" <property name='map'>\n" +
|
||||
" <map>\n" +
|
||||
" <entry key='frontier.uriPrecedencePolicy.basePrecedence' value='1'/>\n" +
|
||||
" </map>\n" +
|
||||
" </property>\n" +
|
||||
"</bean>\n";
|
||||
|
||||
config = config.replace("</beans>", sheets+"</beans>");
|
||||
return config;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import org.archive.crawler.frontier.precedence.PrecedenceLoader;
|
||||
|
||||
|
||||
/**
|
||||
* Tests that precedence values for URIs can be imported from an offline
|
||||
* analysis. This test crawls the same directory structure as
|
||||
* {@link PrecedenceSelfTest1} and expects the URIs to be crawled in the same
|
||||
* order. However, the result is achieved using a
|
||||
* {@link org.archive.crawler.frontier.precedence.PreloadedUriPrecedencePolicy}
|
||||
* to load per-URI precedence information from an external file.
|
||||
*
|
||||
* <p>Such a file could be generated from PageRank analysis of a previously
|
||||
* completed crawl; see {@link http://webteam.archive.org/confluence/display/Heritrix/Offline+PageRank+Analysis+Notes}.
|
||||
* (For this minimal functional test, the PreloadedUriPrecedencePolicy input
|
||||
* file was simply hand-generated.)
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class Precedence2SelfTest extends Precedence1SelfTest {
|
||||
|
||||
|
||||
@Override
|
||||
protected String changeGlobalConfig(String config) {
|
||||
// add an autowired uriPrecedencePolicy with preloaded values
|
||||
String uriPrecedencePolicy =
|
||||
" <bean id='uriPrecedencePolicy' class='org.archive.crawler.frontier.precedence.PreloadedUriPrecedencePolicy'>\n" +
|
||||
" <property name='basePrecedence' value='5'/>\n" +
|
||||
" </bean>";
|
||||
config = config.replace("<!--@@BEANS_MOREBEANS@@-->", uriPrecedencePolicy);
|
||||
// suppress superclass insertion of inner bean policy
|
||||
config = config.replace("<!--@@FRONTIER_PROPERTIES@@-->", "");
|
||||
return super.changeGlobalConfig(config);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void configureHeritrix() throws Exception {
|
||||
File src = new File(getJobDir(), "rank.txt");
|
||||
File dest = new File(getJobDir(), "state");
|
||||
String[] args = new String[] {
|
||||
src.getAbsolutePath(),
|
||||
dest.getAbsolutePath()
|
||||
};
|
||||
|
||||
PrecedenceLoader.main(args);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
||||
/**
|
||||
* Tests that URLs can be assigned precedence values based on in-line analysis
|
||||
* module that prioritizes newly discovered links based on an approximate
|
||||
* ranking during a crawl.
|
||||
*
|
||||
* <p>The test data consists of 15 documents, titled A through O. Each document
|
||||
* links to two other documents, forming a sorted, balanced binary tree:
|
||||
*
|
||||
* <ul>
|
||||
* <li>H</li>
|
||||
* <ul>
|
||||
* <li>D</li>
|
||||
* <ul>
|
||||
* <li>B</li>
|
||||
* <ul>
|
||||
* <li>A</li>
|
||||
* <li>C</li>
|
||||
* </ul>
|
||||
* <li>F</li>
|
||||
* <ul>
|
||||
* <li>E</li>
|
||||
* <li>G</li>
|
||||
* </ul>
|
||||
* </ul>
|
||||
* <li>L</li>
|
||||
* <ul>
|
||||
* <li>J</li>
|
||||
* <ul>
|
||||
* <li>I</li>
|
||||
* <li>K</li>
|
||||
* </ul>
|
||||
* <li>N</li>
|
||||
* <ul>
|
||||
* <li>M</li>
|
||||
* <li>O</li>
|
||||
* </ul>
|
||||
* </ul>
|
||||
* </ul>
|
||||
* </ul>
|
||||
*
|
||||
* <p>If H is the seed, then Heritrix would ordinarily crawl these in the order
|
||||
* <code>(H, L, D, J, N, F, B, K, I, M, O, G, E, C, A)</code> -- loosely the
|
||||
* order the links were discovered.
|
||||
*
|
||||
* <p>However, this test uses the {@link KeyWordProcessor} to ensure that, if
|
||||
* a document contains a certain keyword, then that document's out links are
|
||||
* crawled before the out links of documents that do not contain the keyword.
|
||||
*
|
||||
* <p>The documents A, B, D, E, H, I, J and M all contain the keyword (these
|
||||
* are the "left-branch"/first-link documents in the tree above, plus the
|
||||
* root/seed). The other documents do not.
|
||||
*
|
||||
* <p>Therefore this test makes sure that the children of documents containing
|
||||
* the keyword are crawled before children of documents not containing the
|
||||
* keyword:
|
||||
*
|
||||
* <ol>
|
||||
* <li>The children of D (B and F) should be crawled before the children
|
||||
* of L (J and N).</li>
|
||||
* <li>The children of B (A and C) should be crawled before the children
|
||||
* of F (E and G).</li>
|
||||
* <li>The children of J (I and K) should be crawled before the children of
|
||||
* N (M and O).
|
||||
* </ol>
|
||||
*
|
||||
* <p>This test provides a simple proof-of-concept that shows how the content
|
||||
* of one URI can alter the precedence of the out links of that URI. See
|
||||
* {@link KeyWordProcessor} for suggestions on more sophisticated approaches.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class Precedence3SelfTest extends SelfTestBase {
|
||||
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
File crawlLog = new File(getLogsDir(), "crawl.log");
|
||||
BufferedReader br = null;
|
||||
List<String> crawled = new ArrayList<String>();
|
||||
|
||||
try {
|
||||
br = new BufferedReader(new FileReader(crawlLog));
|
||||
for (String s = br.readLine(); s != null; s = br.readLine()) {
|
||||
s = s.substring(42);
|
||||
int i = s.indexOf(' ');
|
||||
s = s.substring(0, i);
|
||||
crawled.add(s);
|
||||
}
|
||||
} finally {
|
||||
IOUtils.closeQuietly(br);
|
||||
}
|
||||
|
||||
System.out.println(crawled);
|
||||
|
||||
// assertEquals("dns:localhost", crawled.get(0));
|
||||
assertEquals("http://127.0.0.1:7777/robots.txt", crawled.get(0));
|
||||
assertEquals("http://127.0.0.1:7777/H.html", crawled.get(1));
|
||||
|
||||
// D contains the keyword and L does not.
|
||||
// D's children (B and F) should be crawled before L's (J and N).
|
||||
assertBefore(crawled, 'B', 'F', 'J', 'N');
|
||||
|
||||
// B contains the keyword and F does not.
|
||||
// B's children (A and C) should be crawled before F's (E and G).
|
||||
assertBefore(crawled, 'A', 'C', 'E', 'G');
|
||||
|
||||
// J contains the keyword and N does not.
|
||||
// J's children (I and K) should be crawled before N's (M and O).
|
||||
assertBefore(crawled, 'I', 'K', 'M', 'O');
|
||||
}
|
||||
|
||||
|
||||
private boolean assertBefore(List<String> crawled,
|
||||
char k1, char k2, char n1, char n2) {
|
||||
int k1Index = crawled.indexOf(toFullURI(k1));
|
||||
int k2Index = crawled.indexOf(toFullURI(k2));
|
||||
int n1Index = crawled.indexOf(toFullURI(n1));
|
||||
int n2Index = crawled.indexOf(toFullURI(n2));
|
||||
// Make sure all four documents were actually crawled.
|
||||
assertTrue(k1Index > 0);
|
||||
assertTrue(k2Index > 0);
|
||||
assertTrue(n1Index > 0);
|
||||
assertTrue(n2Index > 0);
|
||||
|
||||
// Make sure children of keyword-containing-parent were crawled before
|
||||
// children of no-keyword-containing-parent.
|
||||
assertTrue(k1Index + " >= " + n1Index, k1Index < n1Index);
|
||||
assertTrue(k1Index + " >= " + n2Index, k1Index < n2Index);
|
||||
assertTrue(k2Index + " >= " + n1Index, k2Index < n1Index);
|
||||
assertTrue(k2Index + " >= " + n1Index, k2Index < n2Index);
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private String toFullURI(char ch) {
|
||||
return "http://127.0.0.1:7777/" + ch + ".html";
|
||||
}
|
||||
|
||||
protected String getSeedsString() {
|
||||
return "http://127.0.0.1:7777/H.html";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String changeGlobalConfig(String config) {
|
||||
// single toethread
|
||||
config = config.replace("@@MORE_PROPERTIES@@", "crawlController.maxToeThreads=1");
|
||||
|
||||
// add the keyword-based uriPrecedencePolicy
|
||||
String uriPrecedencePolicy =
|
||||
"<property name='uriPrecedencePolicy'>\n" +
|
||||
" <bean class='org.archive.crawler.selftest.KeyWordUriPrecedencePolicy'>\n" +
|
||||
" <property name='basePrecedence' value='5'/>\n" +
|
||||
" </bean>" +
|
||||
"</property>";
|
||||
config = config.replace("<!--@@FRONTIER_PROPERTIES@@-->", uriPrecedencePolicy);
|
||||
|
||||
// add the keyword processor after linkScoper
|
||||
config = config.replace(
|
||||
"<ref bean=\"linksScoper\"/>",
|
||||
"<ref bean=\"linksScoper\"/>\n" +
|
||||
"<bean class='org.archive.crawler.selftest.KeyWordProcessor'/>");
|
||||
return super.changeGlobalConfig(config);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
||||
/**
|
||||
* Tests that operators can manually assign precedence values to individual
|
||||
* URLs.
|
||||
*
|
||||
* <p>This class crawls the same directory structure as
|
||||
* {@link Precedence1SelfTest}, using the same number of sheets. However,
|
||||
* insteading of creating groups of URIs using SURT prefixes, the HiPri and
|
||||
* LoPri sheets are assigned to two individual URIs. The test then assures
|
||||
* that the HiPri URI is crawled before anything else, and that the LoPri
|
||||
* URL is crawled after everything else.
|
||||
*
|
||||
* @author pjack
|
||||
*/
|
||||
public class Precedence4SelfTest extends Precedence1SelfTest {
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
File crawlLog = new File(getLogsDir(), "crawl.log");
|
||||
BufferedReader br = null;
|
||||
List<String> crawled = new ArrayList<String>();
|
||||
try {
|
||||
br = new BufferedReader(new FileReader(crawlLog));
|
||||
for (String s = br.readLine(); s != null; s = br.readLine()) {
|
||||
s = s.substring(42);
|
||||
int i = s.indexOf(' ');
|
||||
s = s.substring(0, i);
|
||||
crawled.add(s);
|
||||
}
|
||||
} finally {
|
||||
IOUtils.closeQuietly(br);
|
||||
}
|
||||
|
||||
//assertEquals("dns:localhost", crawled.get(0));
|
||||
assertEquals("http://127.0.0.1:7777/robots.txt", crawled.get(0));
|
||||
assertEquals("http://127.0.0.1:7777/five/a.html", crawled.get(1));
|
||||
assertEquals("http://127.0.0.1:7777/five/b.html", crawled.get(crawled.size() - 1));
|
||||
}
|
||||
|
||||
|
||||
|
||||
protected String getSeedsString() {
|
||||
return "http://127.0.0.1:7777/seed.html\\n"+
|
||||
"http://127.0.0.1:7777/one/a.html\\n"+
|
||||
"http://127.0.0.1:7777/five/a.html\\n"+
|
||||
"http://127.0.0.1:7777/ten/a.html\\n"+
|
||||
"http://127.0.0.1:7777/ten/b.html\\n"+
|
||||
"http://127.0.0.1:7777/five/b.html\\n"+
|
||||
"http://127.0.0.1:7777/one/b.html\\n"+
|
||||
"http://127.0.0.1:7777/five/c.html\\n"+
|
||||
"http://127.0.0.1:7777/one/c.html\\n"+
|
||||
"http://127.0.0.1:7777/ten/c.html";
|
||||
}
|
||||
|
||||
protected String configureSheets(String config) {
|
||||
// add sheets which overlay alternate precedence values for two
|
||||
// specific URIs
|
||||
String sheets =
|
||||
"<bean id='loPri' class='org.archive.crawler.spring.SheetForSurtPrefixes'>\n" +
|
||||
" <property name='surtPrefixes'>\n" +
|
||||
" <list>\n" +
|
||||
" <value>http://(127.0.0.1:7777)/five/b.html</value>\n" +
|
||||
" </list>\n" +
|
||||
" </property>\n" +
|
||||
" <property name='map'>\n" +
|
||||
" <map>\n" +
|
||||
" <entry key='frontier.uriPrecedencePolicy.basePrecedence' value='10'/>\n" +
|
||||
" </map>\n" +
|
||||
" </property>\n" +
|
||||
"</bean>\n" +
|
||||
"<bean id='hiPri' class='org.archive.crawler.spring.SheetForSurtPrefixes'>\n" +
|
||||
" <property name='surtPrefixes'>\n" +
|
||||
" <list>\n" +
|
||||
" <value>http://(127.0.0.1:7777)/five/a.html</value>\n" +
|
||||
" </list>\n" +
|
||||
" </property>\n" +
|
||||
" <property name='map'>\n" +
|
||||
" <map>\n" +
|
||||
" <entry key='frontier.uriPrecedencePolicy.basePrecedence' value='1'/>\n" +
|
||||
" </map>\n" +
|
||||
" </property>\n" +
|
||||
"</bean>\n";
|
||||
|
||||
config = config.replace("</beans>", sheets+"</beans>");
|
||||
return config;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* RandomServlet.java
|
||||
*
|
||||
* Created on Feb 28, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Writer;
|
||||
import java.util.Random;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
|
||||
/**
|
||||
* @author pjack
|
||||
*
|
||||
*/
|
||||
public class RandomServlet extends HttpServlet {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
|
||||
private int maxHops = 3;
|
||||
|
||||
private int minPort = 7000;
|
||||
|
||||
private int maxPort = 7010;
|
||||
|
||||
private String host = "localhost";
|
||||
|
||||
private String pathRoot = "random";
|
||||
|
||||
|
||||
public RandomServlet() {
|
||||
}
|
||||
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
|
||||
public int getMaxHops() {
|
||||
return maxHops;
|
||||
}
|
||||
|
||||
|
||||
public void setMaxHops(int maxHops) {
|
||||
this.maxHops = maxHops;
|
||||
}
|
||||
|
||||
|
||||
public int getMaxPort() {
|
||||
return maxPort;
|
||||
}
|
||||
|
||||
|
||||
public void setMaxPort(int maxPort) {
|
||||
this.maxPort = maxPort;
|
||||
}
|
||||
|
||||
|
||||
public int getMinPort() {
|
||||
return minPort;
|
||||
}
|
||||
|
||||
|
||||
public void setMinPort(int minPort) {
|
||||
this.minPort = minPort;
|
||||
}
|
||||
|
||||
|
||||
public String getPathRoot() {
|
||||
return pathRoot;
|
||||
}
|
||||
|
||||
|
||||
public void setPathRoot(String pathRoot) {
|
||||
this.pathRoot = pathRoot;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
|
||||
throws ServletException, IOException {
|
||||
resp.setContentType("text/html");
|
||||
RandomServletLinkWriter rslw = new RandomServletLinkWriter();
|
||||
|
||||
rslw.setHost(host);
|
||||
rslw.setPathRoot(pathRoot);
|
||||
rslw.setMaxHops(maxHops);
|
||||
rslw.setPortRange(minPort, maxPort);
|
||||
rslw.setPathInfo(req.getPathInfo());
|
||||
rslw.write(resp.getWriter());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
class RandomServletLinkWriter {
|
||||
|
||||
|
||||
final private static long SEED_BASE = -4739205649012677248L;
|
||||
|
||||
final private static int MAX_LINKS = 50;
|
||||
|
||||
String host;
|
||||
String pathRoot;
|
||||
|
||||
int pathValue;
|
||||
Random random;
|
||||
Writer writer;
|
||||
int maxHops;
|
||||
int minPort;
|
||||
int maxPort;
|
||||
|
||||
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
|
||||
public void setPathRoot(String pathRoot) {
|
||||
this.pathRoot = pathRoot;
|
||||
}
|
||||
|
||||
|
||||
public void setMaxHops(int maxHops) {
|
||||
this.maxHops = maxHops;
|
||||
}
|
||||
|
||||
|
||||
public void setPortRange(int min, int max) {
|
||||
this.minPort = min;
|
||||
this.maxPort = max;
|
||||
}
|
||||
|
||||
|
||||
public void setPathInfo(String pathInfo) {
|
||||
this.pathValue = fromPath(pathInfo);
|
||||
long seed = SEED_BASE * (long)pathValue;
|
||||
this.random = new Random(seed);
|
||||
}
|
||||
|
||||
|
||||
public void write(Writer writer) throws IOException {
|
||||
this.writer = writer;
|
||||
|
||||
for (int i = minPort; i < maxPort; i++) {
|
||||
writePortLinks(i);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void writePortLinks(int port) throws IOException {
|
||||
writeLink(port, pathValue - 1);
|
||||
writeLink(port, pathValue + 1);
|
||||
|
||||
int max = random.nextInt(MAX_LINKS);
|
||||
for (int i = 0; i < max; i++) {
|
||||
writeLink(port, random.nextInt(max(maxHops)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private int max(int digits) {
|
||||
int r = 1;
|
||||
for (; digits > 0; digits--) {
|
||||
r *= 10;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
|
||||
private void writeLink(int port, int value) throws IOException {
|
||||
if (value < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (value >= max(maxHops)) {
|
||||
return;
|
||||
}
|
||||
|
||||
String path = toPath(value);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("<a href=\"http://").append(host).append(':').append(port);
|
||||
sb.append('/').append(pathRoot).append('/').append(path);
|
||||
sb.append("\">link ").append(value).append("</a>\n");
|
||||
writer.write(sb.toString());
|
||||
writer.flush();
|
||||
}
|
||||
|
||||
|
||||
public static String toPath(int value) {
|
||||
if (value < 0) {
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
if (value == 0) {
|
||||
return "0";
|
||||
}
|
||||
String r = "";
|
||||
while (value > 0) {
|
||||
r += (value % 10) + "/";
|
||||
value = value / 10;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
|
||||
public static int fromPath(String path) {
|
||||
if (path.startsWith("/")) {
|
||||
path = path.substring(1);
|
||||
}
|
||||
if (path.length() == 0) {
|
||||
return 0;
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
String[] digits = path.split("/");
|
||||
for (int i = 0; i < digits.length; i++) {
|
||||
sb.insert(0, digits[i].charAt(0));
|
||||
}
|
||||
return Integer.parseInt(sb.toString());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* RandomServletTest.java
|
||||
*
|
||||
* Created on Feb 28, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
/**
|
||||
* @author pjack
|
||||
*
|
||||
*/
|
||||
public class RandomServletTest extends TestCase {
|
||||
|
||||
|
||||
|
||||
public void testPathParse() {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
String s = RandomServletLinkWriter.toPath(i);
|
||||
// System.out.println(i +" -> " + s);
|
||||
int v = RandomServletLinkWriter.fromPath(s);
|
||||
int v2 = RandomServletLinkWriter.fromPath("/" + s);
|
||||
assertEquals(i, v);
|
||||
assertEquals(i, v2);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,369 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.math.BigInteger;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.archive.crawler.Heritrix;
|
||||
import org.archive.io.ArchiveRecordHeader;
|
||||
import org.archive.io.arc.ARCReaderFactory;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.TmpDirTestCase;
|
||||
import org.mortbay.jetty.Handler;
|
||||
import org.mortbay.jetty.Server;
|
||||
import org.mortbay.jetty.bio.SocketConnector;
|
||||
import org.mortbay.jetty.handler.DefaultHandler;
|
||||
import org.mortbay.jetty.handler.HandlerList;
|
||||
import org.mortbay.jetty.handler.ResourceHandler;
|
||||
|
||||
/**
|
||||
* Base class for 'self tests', integrations tests formatted as unit
|
||||
* tests, where the crawler launches an entire crawl exercising multiple
|
||||
* features against a test harness website.
|
||||
*
|
||||
* @contributor pjack
|
||||
* @contributor gojomo
|
||||
*/
|
||||
public abstract class SelfTestBase extends TmpDirTestCase {
|
||||
|
||||
final private Logger LOGGER =
|
||||
Logger.getLogger(SelfTestBase.class.getName());
|
||||
|
||||
protected Heritrix heritrix;
|
||||
protected Server httpServer;
|
||||
|
||||
protected void open() throws Exception {
|
||||
// We expect to be run from the project directory.
|
||||
// (Both eclipse and maven run junit tests from there).
|
||||
String name = getSelfTestName();
|
||||
|
||||
// Make sure the project directory contains a selftest profile
|
||||
// and content for the self test.
|
||||
File src = getTestDataDir();
|
||||
if (!src.exists()) {
|
||||
throw new Exception("No selftest directory for " + name);
|
||||
}
|
||||
|
||||
// Create temporary directories for Heritrix to run in.
|
||||
File tmpDir = new File(getTmpDir(), "selftest");
|
||||
File tmpTestDir = new File(tmpDir, name);
|
||||
|
||||
// If we have an old job lying around from a previous run, delete it.
|
||||
File tmpJobs = new File(tmpTestDir, "jobs");
|
||||
if (tmpJobs.exists()) {
|
||||
FileUtils.deleteDirectory(tmpJobs);
|
||||
}
|
||||
|
||||
// Copy the selftest's profile in the project directory to the
|
||||
// default profile in the temporary Heritrix directory.
|
||||
File tmpDefProfile = new File(tmpJobs, "selftest-job");
|
||||
org.apache.commons.io.FileUtils.copyDirectory(new File(src, "profile"), tmpDefProfile);
|
||||
|
||||
// Start up a Jetty that serves the selftest's content directory.
|
||||
startHttpServer();
|
||||
|
||||
// Copy configuration for eg Logging over
|
||||
File tmpConfDir = new File(tmpTestDir, "conf");
|
||||
tmpConfDir.mkdirs();
|
||||
File srcConf = new File(src.getParentFile(), "conf");
|
||||
FileUtils.copyDirectory(srcConf, tmpConfDir);
|
||||
|
||||
String crawlerBeansText = FileUtils.readFileToString(
|
||||
new File(srcConf, "selftest-crawler-beans.cxml"));
|
||||
crawlerBeansText = changeGlobalConfig(crawlerBeansText);
|
||||
File crawlerBeans = new File(tmpDefProfile, "selftest-crawler-beans.cxml");
|
||||
FileWriter fw = new FileWriter(crawlerBeans);
|
||||
fw.write(crawlerBeansText);
|
||||
fw.close();
|
||||
|
||||
startHeritrix(tmpTestDir.getAbsolutePath());
|
||||
|
||||
waitForCrawlFinish();
|
||||
}
|
||||
|
||||
|
||||
protected String changeGlobalConfig(String config) {
|
||||
config = config.replace(
|
||||
"@@URL_VALUE@@","http://crawler.archive.org/selftestcrawl");
|
||||
// if not already changed, used default self-test start URL
|
||||
config = config.replace(
|
||||
"@@SEEDS_VALUE@@", getSeedsString());
|
||||
// if not already replaced, remove other placeholder
|
||||
config = config.replace("@@MORE_PROPERTIES@@","");
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get seeds for this test. Should be in form that can be
|
||||
* spliced into a Java properties-format string (any internal
|
||||
* lineends escaped with '\').
|
||||
* @return String seeds to use
|
||||
*/
|
||||
protected String getSeedsString() {
|
||||
// default barring overrides
|
||||
return "http://127.0.0.1:7777/index.html";
|
||||
}
|
||||
|
||||
|
||||
protected void close() throws Exception {
|
||||
stopHttpServer();
|
||||
stopHeritrix();
|
||||
}
|
||||
|
||||
public void testSomething() throws Exception {
|
||||
try {
|
||||
boolean fail = false;
|
||||
try {
|
||||
open();
|
||||
verifyCommon();
|
||||
verify();
|
||||
} finally {
|
||||
try {
|
||||
close();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail = true;
|
||||
}
|
||||
}
|
||||
assertFalse(fail);
|
||||
} catch (Exception e) {
|
||||
// I hate maven.
|
||||
e.printStackTrace();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected abstract void verify() throws Exception;
|
||||
|
||||
|
||||
protected void stopHttpServer() throws Exception {
|
||||
try {
|
||||
httpServer.stop();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void startHttpServer() throws Exception {
|
||||
Server server = new Server();
|
||||
SocketConnector sc = new SocketConnector();
|
||||
sc.setHost("localhost");
|
||||
sc.setPort(7777);
|
||||
server.addConnector(sc);
|
||||
ResourceHandler rhandler = new ResourceHandler();
|
||||
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
|
||||
|
||||
HandlerList handlers = new HandlerList();
|
||||
handlers.setHandlers(new Handler[] { rhandler, new DefaultHandler() });
|
||||
server.setHandler(handlers);
|
||||
|
||||
this.httpServer = server;
|
||||
server.start();
|
||||
}
|
||||
|
||||
|
||||
protected void startHeritrix(String path) throws Exception {
|
||||
String authPassword =
|
||||
(new BigInteger(SecureRandom.getSeed(16))).toString(16);
|
||||
String[] args = { "-j", path + "/jobs", "-a", authPassword };
|
||||
// TODO: add auth password?
|
||||
heritrix = new Heritrix();
|
||||
heritrix.instanceMain(args);
|
||||
|
||||
configureHeritrix();
|
||||
|
||||
heritrix.getEngine().requestLaunch("selftest-job");
|
||||
}
|
||||
|
||||
|
||||
protected void configureHeritrix() throws Exception {
|
||||
// by default do nothing
|
||||
}
|
||||
|
||||
|
||||
protected void stopHeritrix() throws Exception {
|
||||
heritrix.getEngine().shutdown();
|
||||
heritrix.getComponent().stop();
|
||||
}
|
||||
|
||||
protected void waitForCrawlFinish() throws Exception {
|
||||
heritrix.getEngine().waitForNoRunningJobs(0);
|
||||
}
|
||||
|
||||
protected File getSrcHtdocs() {
|
||||
return new File(getTestDataDir(), "htdocs");
|
||||
}
|
||||
|
||||
protected File getTestDataDir() {
|
||||
File r = new File("testdata");
|
||||
if (!r.exists()) {
|
||||
r = new File("engine");
|
||||
r = new File(r, "testdata");
|
||||
if (!r.exists()) {
|
||||
throw new IllegalStateException(
|
||||
"Can't find selfest testdata " +
|
||||
"(tried testdata/selftest and " +
|
||||
"heritrix/testdata/selftest)");
|
||||
}
|
||||
}
|
||||
r = new File(r, "selftest");
|
||||
r = new File(r, getSelfTestName());
|
||||
if (!r.exists()) {
|
||||
throw new IllegalStateException("No testdata directory: "
|
||||
+ r.getAbsolutePath());
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
|
||||
protected File getCrawlDir() {
|
||||
File tmp = getTmpDir();
|
||||
File selftest = new File(tmp, "selftest");
|
||||
File crawl = new File(selftest, getSelfTestName());
|
||||
return crawl;
|
||||
}
|
||||
|
||||
protected File getJobDir() {
|
||||
File crawl = getCrawlDir();
|
||||
File jobs = new File(crawl, "jobs");
|
||||
File theJob = new File(jobs, "selftest-job");
|
||||
return theJob;
|
||||
}
|
||||
|
||||
|
||||
protected File getArcDir() {
|
||||
return new File(getJobDir(), "arcs");
|
||||
}
|
||||
|
||||
|
||||
protected File getLogsDir() {
|
||||
return new File(getJobDir(), "logs");
|
||||
}
|
||||
|
||||
|
||||
|
||||
private String getSelfTestName() {
|
||||
String full = getClass().getName();
|
||||
int i = full.lastIndexOf('.');
|
||||
return full.substring(i + 1);
|
||||
}
|
||||
|
||||
protected void verifyArcsClosed() {
|
||||
File arcsDir = getArcDir();
|
||||
if (!arcsDir.exists()) {
|
||||
throw new IllegalStateException("Missing arc dir " +
|
||||
arcsDir.getAbsolutePath());
|
||||
}
|
||||
for (File f: arcsDir.listFiles()) {
|
||||
String fn = f.getName();
|
||||
if (fn.endsWith(".open")) {
|
||||
throw new IllegalStateException(
|
||||
"Arc file not closed at end of crawl: " + f.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected void verifyLogFileEmpty(String logFileName) {
|
||||
File logsDir = getLogsDir();
|
||||
File log = new File(logsDir, logFileName);
|
||||
if (log.length() != 0) {
|
||||
throw new IllegalStateException("Log " + logFileName +
|
||||
" isn't empty.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void verifyCommon() throws Exception {
|
||||
verifyLogFileEmpty("uri-errors.log");
|
||||
verifyLogFileEmpty("runtime-errors.log");
|
||||
verifyLogFileEmpty("local-errors.log");
|
||||
verifyProgressStatistics();
|
||||
verifyArcsClosed();
|
||||
}
|
||||
|
||||
|
||||
protected void verifyProgressStatistics() throws IOException {
|
||||
File logs = new File(getJobDir(), "logs");
|
||||
File statsFile = new File(logs, "progress-statistics.log");
|
||||
String stats = FileUtils.readFileToString(statsFile);
|
||||
if (!stats.contains("CRAWL RESUMED - Preparing")) {
|
||||
fail("progress-statistics.log has no Prepared line.");
|
||||
}
|
||||
if (!stats.contains("CRAWL RESUMED - Running")) {
|
||||
fail("progress-statistics.log has no Running line.");
|
||||
}
|
||||
if (!stats.contains("CRAWL ENDING - Finished")) {
|
||||
fail("progress-statistics.log has missing/wrong Finished line.");
|
||||
}
|
||||
if (!stats.contains("doc/s(avg)")) {
|
||||
fail("progress-statistics.log has no legend.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected List<ArchiveRecordHeader> headersInArcs() throws IOException {
|
||||
List<ArchiveRecordHeader> result = new ArrayList<ArchiveRecordHeader>();
|
||||
File arcsDir = getArcDir();
|
||||
if (!arcsDir.exists()) {
|
||||
throw new IllegalStateException("Missing arc dir " +
|
||||
arcsDir.getAbsolutePath());
|
||||
}
|
||||
File[] files = arcsDir.listFiles();
|
||||
if (files == null) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
for (File f: files) {
|
||||
result.addAll(ARCReaderFactory.get(f).validate());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
protected Set<String> filesInArcs() throws IOException {
|
||||
List<ArchiveRecordHeader> headers = headersInArcs();
|
||||
HashSet<String> result = new HashSet<String>();
|
||||
for (ArchiveRecordHeader arh: headers) {
|
||||
UURI uuri = UURIFactory.getInstance(arh.getUrl());
|
||||
String path = uuri.getPath();
|
||||
if (path.startsWith("/")) {
|
||||
path = path.substring(1);
|
||||
}
|
||||
if (arh.getUrl().startsWith("http:")) {
|
||||
result.add(path);
|
||||
}
|
||||
}
|
||||
LOGGER.finest(result.toString());
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* SimpleSelfTest.java
|
||||
*
|
||||
* Created on Feb 22, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
/**
|
||||
* @contributor pjack
|
||||
* @contributor gojomo
|
||||
*/
|
||||
public class SimpleSelfTest extends SelfTestBase {
|
||||
|
||||
|
||||
final private static Set<String> EXPECTED = Collections.unmodifiableSet(
|
||||
new HashSet<String>(Arrays.asList(new String[] {
|
||||
"index.html", "link1.html", "link2.html", "link3.html", "robots.txt"
|
||||
})));
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
Set<String> files = filesInArcs();
|
||||
assertTrue(EXPECTED.equals(files));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (C) 2007 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
*
|
||||
* UserAgentSelfTest.java
|
||||
*
|
||||
* Created on Apr 27, 2007
|
||||
*
|
||||
* $Id:$
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import org.archive.util.ArchiveUtils;
|
||||
import org.mortbay.jetty.Handler;
|
||||
import org.mortbay.jetty.Server;
|
||||
import org.mortbay.jetty.bio.SocketConnector;
|
||||
import org.mortbay.jetty.handler.DefaultHandler;
|
||||
import org.mortbay.jetty.handler.HandlerList;
|
||||
import org.mortbay.jetty.handler.ResourceHandler;
|
||||
import org.mortbay.jetty.servlet.ServletHandler;
|
||||
import org.mortbay.jetty.servlet.ServletHolder;
|
||||
|
||||
/**
|
||||
* @author pjack
|
||||
*
|
||||
*/
|
||||
public class UserAgentSelfTest extends SelfTestBase {
|
||||
|
||||
|
||||
private UserAgentServlet servlet;
|
||||
|
||||
|
||||
final private static String EXPECTED_UA =
|
||||
"Mozilla/5.0 (compatible; heritrix/" + ArchiveUtils.VERSION
|
||||
+ " +http://crawler.archive.org/selftestcrawl)";
|
||||
|
||||
@Override
|
||||
protected void verify() throws Exception {
|
||||
assertEquals(EXPECTED_UA, servlet.getUserAgent());
|
||||
// assertEquals(EXPECTED_FROM, servlet.getFrom());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void startHttpServer() throws Exception {
|
||||
Server server = new Server();
|
||||
|
||||
SocketConnector sc = new SocketConnector();
|
||||
sc.setHost("localhost");
|
||||
sc.setPort(7777);
|
||||
server.addConnector(sc);
|
||||
ResourceHandler rhandler = new ResourceHandler();
|
||||
rhandler.setResourceBase(getSrcHtdocs().getAbsolutePath());
|
||||
|
||||
ServletHandler servletHandler = new ServletHandler();
|
||||
|
||||
HandlerList handlers = new HandlerList();
|
||||
handlers.setHandlers(new Handler[] {
|
||||
rhandler,
|
||||
servletHandler,
|
||||
new DefaultHandler() });
|
||||
server.setHandler(handlers);
|
||||
|
||||
this.servlet = new UserAgentServlet();
|
||||
ServletHolder holder = new ServletHolder(servlet);
|
||||
servletHandler.addServletWithMapping(holder, "/*");
|
||||
|
||||
this.httpServer = server;
|
||||
this.httpServer.start();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.selftest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Enumeration;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServlet;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
/**
|
||||
* @author pjack
|
||||
*/
|
||||
public class UserAgentServlet extends HttpServlet {
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
||||
private String ua;
|
||||
private String from;
|
||||
|
||||
@Override
|
||||
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
|
||||
throws ServletException, IOException {
|
||||
Enumeration<?> e = req.getHeaderNames();
|
||||
while (e.hasMoreElements()) {
|
||||
String name = (String)e.nextElement();
|
||||
System.out.println(name + "=" + req.getHeader(name));
|
||||
}
|
||||
this.ua = req.getHeader("User-Agent");
|
||||
this.from = req.getHeader("From");
|
||||
resp.getWriter().println("This space intentionally left blank.");
|
||||
resp.getWriter().close();
|
||||
}
|
||||
|
||||
|
||||
public String getUserAgent() {
|
||||
return ua;
|
||||
}
|
||||
|
||||
|
||||
public String getFrom() {
|
||||
return from;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
|
||||
<html>
|
||||
<head>
|
||||
<title>org.archive.crawler.selftest package</title>
|
||||
</head>
|
||||
<body>Provides the client-side aspect of the heritrix integration self test.
|
||||
<p>The <i>selftest</i> webapp is the repository for the serverside of the
|
||||
intergration test.</p> <p>The integration self test is run from the command
|
||||
line. Invocation makes the crawler go up against itself trawling the
|
||||
<i>selftest</i> webapp. When done, the product -- arc and log files -- are
|
||||
analyzed by code herein to verify test pass or fail.</p> <p>The integration
|
||||
self test is the aggregation of multiple individual tests each testing a
|
||||
particular crawler aspect. For example, the <i>Robots</i> test validates
|
||||
the crawler's parse of <i>robots.txt</i>. Each test comprises a directory
|
||||
under the <i>selftest</i> webapp named for the test into which we put the
|
||||
server pages that express the scenario to test, and a class from this
|
||||
package named for test webapp directory w/ a <code>SelfTest</code> suffix.
|
||||
The selftest class verifies test success. Each selftest class subclasses
|
||||
<code>org.archive.crawler.selftest.SelfTestCase</code> which is itself
|
||||
a subclass of <code>org.junit.TestCase</code>). All tests need to be
|
||||
registered with the {@link org.archive.crawler.selftest.AllSelfTestCases}
|
||||
class and must live in the org.archive.crawler.selftest package. The class
|
||||
{@link org.archive.crawler.selftest.SelfTestCrawlJobHandler}
|
||||
manages the running of selftest.</p>
|
||||
<p>Run one test only by passing its name as the option value to the
|
||||
selftest argument.</p>
|
||||
<p>The first crop of self tests are
|
||||
derived from tests developed by Parker Thompson < pt at archive dot org
|
||||
>. See <a
|
||||
href="http://cvs.sourceforge.net/viewcvs.py/archive-crawler/Tests/">Tests</a>.
|
||||
These tests in turn look to have been derived from <a
|
||||
href="http://www.searchtools.com/test/">Testing Search Indexing
|
||||
Systems1</a>. </p> <h3>Adding a Self Test</h3> <p>TODO</p> <h2>Related
|
||||
Documentation</h2> <p>TODO</p> </body>
|
||||
</html>
|
||||
@@ -0,0 +1,210 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.util;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import junit.framework.Test;
|
||||
import junit.framework.TestSuite;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.archive.crawler.datamodel.CrawlURI;
|
||||
import org.archive.crawler.datamodel.UriUniqFilter;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.TmpDirTestCase;
|
||||
|
||||
import com.sleepycat.je.DatabaseException;
|
||||
|
||||
|
||||
/**
|
||||
* Test BdbUriUniqFilter.
|
||||
* @author stack
|
||||
*/
|
||||
public class BdbUriUniqFilterTest extends TmpDirTestCase
|
||||
implements UriUniqFilter.CrawlUriReceiver {
|
||||
private Logger logger =
|
||||
Logger.getLogger(BdbUriUniqFilterTest.class.getName());
|
||||
|
||||
private UriUniqFilter filter = null;
|
||||
private File bdbDir = null;
|
||||
|
||||
/**
|
||||
* Set to true if we visited received.
|
||||
*/
|
||||
private boolean received = false;
|
||||
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
// Remove any bdb that already exists.
|
||||
this.bdbDir = new File(getTmpDir(), this.getClass().getName());
|
||||
if (this.bdbDir.exists()) {
|
||||
FileUtils.deleteDirectory(bdbDir);
|
||||
}
|
||||
this.filter = new BdbUriUniqFilter(bdbDir, 50);
|
||||
this.filter.setDestination(this);
|
||||
}
|
||||
|
||||
protected void tearDown() throws Exception {
|
||||
super.tearDown();
|
||||
((BdbUriUniqFilter)this.filter).close();
|
||||
// if (this.bdbDir.exists()) {
|
||||
// FileUtils.deleteDir(bdbDir);
|
||||
// }
|
||||
}
|
||||
|
||||
public void testAdding() throws URIException {
|
||||
this.filter.add(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
this.filter.addNow(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
this.filter.addForce(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
// Should only have add 'this' once.
|
||||
assertTrue("Count is off", this.filter.count() == 1);
|
||||
}
|
||||
|
||||
public void testCreateKey() {
|
||||
String url = "dns:archive.org";
|
||||
long fingerprint = BdbUriUniqFilter.createKey(url);
|
||||
assertTrue("Fingerprint wrong " + url,
|
||||
fingerprint == 8812917769287344085L);
|
||||
url = "http://archive.org/index.html";
|
||||
fingerprint = BdbUriUniqFilter.createKey(url);
|
||||
assertTrue("Fingerprint wrong " + url,
|
||||
fingerprint == 6613237167064754714L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that two URIs which gave colliding hashes, when previously
|
||||
* the last 40bits of the composite did not sufficiently vary with certain
|
||||
* inputs, no longer collide.
|
||||
*/
|
||||
public void testCreateKeyCollisions() {
|
||||
HashSet<Long> fingerprints = new HashSet<Long>();
|
||||
fingerprints.add(new Long(BdbUriUniqFilter
|
||||
.createKey("dns:mail.daps.dla.mil")));
|
||||
fingerprints.add(new Long(BdbUriUniqFilter
|
||||
.createKey("dns:militaryreview.army.mil")));
|
||||
assertEquals("colliding fingerprints",2,fingerprints.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* Time import of recovery log.
|
||||
* REMOVE
|
||||
* @throws IOException
|
||||
* @throws DatabaseException
|
||||
*/
|
||||
public void testWriting()
|
||||
throws IOException, DatabaseException {
|
||||
long maxcount = 1000;
|
||||
// Look for a system property to override default max count.
|
||||
String key = this.getClass().getName() + ".maxcount";
|
||||
String maxcountStr = System.getProperty(key);
|
||||
logger.info("Looking for override system property " + key);
|
||||
if (maxcountStr != null && maxcountStr.length() > 0) {
|
||||
maxcount = Long.parseLong(maxcountStr);
|
||||
}
|
||||
runTestWriting(maxcount);
|
||||
}
|
||||
|
||||
protected void runTestWriting(long max)
|
||||
throws DatabaseException, URIException {
|
||||
long start = System.currentTimeMillis();
|
||||
ArrayList<UURI> list = new ArrayList<UURI>(1000);
|
||||
int count = 0;
|
||||
for (; count < max; count++) {
|
||||
UURI u = UURIFactory.getInstance("http://www" +
|
||||
count + ".archive.org/" + count + "/index.html");
|
||||
this.filter.add(u.toString(), new CrawlURI(u));
|
||||
if (count > 0 && ((count % 100) == 0)) {
|
||||
list.add(u);
|
||||
}
|
||||
if (count > 0 && ((count % 100000) == 0)) {
|
||||
this.logger.info("Added " + count + " in " +
|
||||
(System.currentTimeMillis() - start) +
|
||||
" misses " +
|
||||
((BdbUriUniqFilter)this.filter).getCacheMisses() +
|
||||
" diff of misses " +
|
||||
((BdbUriUniqFilter)this.filter).getLastCacheMissDiff());
|
||||
}
|
||||
}
|
||||
this.logger.info("Added " + count + " in " +
|
||||
(System.currentTimeMillis() - start));
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
for (Iterator<UURI> i = list.iterator(); i.hasNext();) {
|
||||
UURI uuri = i.next();
|
||||
this.filter.add(uuri.toString(), new CrawlURI(uuri));
|
||||
}
|
||||
this.logger.info("Added random " + list.size() + " in " +
|
||||
(System.currentTimeMillis() - start));
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
for (Iterator<UURI> i = list.iterator(); i.hasNext();) {
|
||||
UURI uuri = i.next();
|
||||
this.filter.add(uuri.toString(), new CrawlURI(uuri));
|
||||
}
|
||||
this.logger.info("Deleted random " + list.size() + " in " +
|
||||
(System.currentTimeMillis() - start));
|
||||
// Looks like delete doesn't work.
|
||||
assertTrue("Count is off: " + this.filter.count(),
|
||||
this.filter.count() == max);
|
||||
}
|
||||
|
||||
public void testNote() {
|
||||
this.filter.note(this.getUri());
|
||||
assertFalse("Receiver was called", this.received);
|
||||
}
|
||||
|
||||
public void testForgetOnEmpty() throws URIException {
|
||||
this.filter.forget(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(getUri())));
|
||||
assertEquals("Didn't forget", 0, this.filter.count());
|
||||
}
|
||||
|
||||
// TODO: Add testForget when non-empty
|
||||
|
||||
public void receive(CrawlURI item) {
|
||||
this.received = true;
|
||||
}
|
||||
|
||||
public String getUri() {
|
||||
return "http://www.archive.org";
|
||||
}
|
||||
|
||||
/**
|
||||
* return the suite of tests for MemQueueTest
|
||||
*
|
||||
* @return the suite of test
|
||||
*/
|
||||
public static Test suite() {
|
||||
return new TestSuite(BdbUriUniqFilterTest.class);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
junit.textui.TestRunner.run(suite());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.util;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.crawler.datamodel.CrawlURI;
|
||||
import org.archive.crawler.datamodel.UriUniqFilter;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
|
||||
|
||||
/**
|
||||
* Test BloomUriUniqFilter.
|
||||
* @author gojomo
|
||||
*/
|
||||
public class BloomUriUniqFilterTest extends TestCase
|
||||
implements UriUniqFilter.CrawlUriReceiver {
|
||||
private Logger logger =
|
||||
Logger.getLogger(BloomUriUniqFilterTest.class.getName());
|
||||
|
||||
private BloomUriUniqFilter filter = null;
|
||||
|
||||
/**
|
||||
* Set to true if we visited received.
|
||||
*/
|
||||
private boolean received = false;
|
||||
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
this.filter = new BloomUriUniqFilter(2000,24);
|
||||
this.filter.setDestination(this);
|
||||
}
|
||||
|
||||
public void testAdding() throws URIException {
|
||||
this.filter.add(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
this.filter.addNow(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
this.filter.addForce(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
// Should only have add 'this' once.
|
||||
assertTrue("Count is off", this.filter.count() == 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test inserting.
|
||||
* @throws URIException
|
||||
* @throws IOException
|
||||
* @throws FileNotFoundException
|
||||
*/
|
||||
public void testWriting() throws URIException {
|
||||
long start = System.currentTimeMillis();
|
||||
ArrayList<UURI> list = new ArrayList<UURI>(1000);
|
||||
int count = 0;
|
||||
final int MAX_COUNT = 1000;
|
||||
for (; count < MAX_COUNT; count++) {
|
||||
assertEquals("count off",count,filter.count());
|
||||
UURI u = UURIFactory.getInstance("http://www" +
|
||||
count + ".archive.org/" + count + "/index.html");
|
||||
assertFalse("already contained "+u.toString(),filter.bloom.contains(u.toString()));
|
||||
logger.fine("adding "+u.toString());
|
||||
filter.add(u.toString(), new CrawlURI(u));
|
||||
assertTrue("not in bloom",filter.bloom.contains(u.toString()));
|
||||
if (count > 0 && ((count % 100) == 0)) {
|
||||
list.add(u);
|
||||
}
|
||||
}
|
||||
logger.fine("Added " + count + " in " +
|
||||
(System.currentTimeMillis() - start));
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
for (Iterator<UURI> i = list.iterator(); i.hasNext();) {
|
||||
UURI uuri = i.next();
|
||||
filter.add(uuri.toString(), new CrawlURI(uuri));
|
||||
}
|
||||
logger.fine("Readded subset " + list.size() + " in " +
|
||||
(System.currentTimeMillis() - start));
|
||||
|
||||
assertTrue("Count is off: " + filter.count(),
|
||||
filter.count() == MAX_COUNT);
|
||||
}
|
||||
|
||||
public void testNote() {
|
||||
filter.note(this.getUri());
|
||||
assertFalse("Receiver was called", this.received);
|
||||
}
|
||||
|
||||
// FORGET CURRENTLY UNSUPPORTED IN BloomUriUniqFilter
|
||||
// public void testForget() throws URIException {
|
||||
// this.filter.forget(this.getUri(),
|
||||
// new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
// assertTrue("Didn't forget", this.filter.count() == 0);
|
||||
// }
|
||||
|
||||
public void receive(CrawlURI item) {
|
||||
this.received = true;
|
||||
}
|
||||
|
||||
public String getUri() {
|
||||
return "http://www.archive.org";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.util;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.commons.httpclient.URIException;
|
||||
import org.archive.crawler.datamodel.CrawlURI;
|
||||
import org.archive.crawler.datamodel.UriUniqFilter;
|
||||
import org.archive.net.UURI;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.archive.util.fingerprint.MemLongFPSet;
|
||||
|
||||
|
||||
/**
|
||||
* Test FPUriUniqFilter.
|
||||
* @author stack
|
||||
*/
|
||||
public class FPUriUniqFilterTest extends TestCase
|
||||
implements UriUniqFilter.CrawlUriReceiver {
|
||||
private Logger logger =
|
||||
Logger.getLogger(FPUriUniqFilterTest.class.getName());
|
||||
|
||||
private UriUniqFilter filter = null;
|
||||
|
||||
/**
|
||||
* Set to true if we visited received.
|
||||
*/
|
||||
private boolean received = false;
|
||||
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
// 17 makes a MemLongFPSet of one meg of longs (64megs).
|
||||
this.filter = new FPUriUniqFilter(new MemLongFPSet(10, 0.75f));
|
||||
this.filter.setDestination(this);
|
||||
}
|
||||
|
||||
public void testAdding() throws URIException {
|
||||
this.filter.add(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
this.filter.addNow(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
this.filter.addForce(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
// Should only have add 'this' once.
|
||||
assertTrue("Count is off", this.filter.count() == 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test inserting and removing.
|
||||
* @throws IOException
|
||||
* @throws FileNotFoundException
|
||||
*/
|
||||
public void testWriting() throws FileNotFoundException, IOException {
|
||||
long start = System.currentTimeMillis();
|
||||
ArrayList<UURI> list = new ArrayList<UURI>(1000);
|
||||
int count = 0;
|
||||
final int MAX_COUNT = 1000;
|
||||
for (; count < MAX_COUNT; count++) {
|
||||
UURI u = UURIFactory.getInstance("http://www" +
|
||||
count + ".archive.org/" + count + "/index.html");
|
||||
this.filter.add(u.toString(), new CrawlURI(u));
|
||||
if (count > 0 && ((count % 100) == 0)) {
|
||||
list.add(u);
|
||||
}
|
||||
}
|
||||
this.logger.info("Added " + count + " in " +
|
||||
(System.currentTimeMillis() - start));
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
for (Iterator<UURI> i = list.iterator(); i.hasNext();) {
|
||||
UURI uuri = i.next();
|
||||
this.filter.add(uuri.toString(), new CrawlURI(uuri));
|
||||
}
|
||||
this.logger.info("Added random " + list.size() + " in " +
|
||||
(System.currentTimeMillis() - start));
|
||||
|
||||
start = System.currentTimeMillis();
|
||||
for (Iterator<UURI> i = list.iterator(); i.hasNext();) {
|
||||
UURI uuri = i.next();
|
||||
this.filter.add(uuri.toString(), new CrawlURI(uuri));
|
||||
}
|
||||
this.logger.info("Deleted random " + list.size() + " in " +
|
||||
(System.currentTimeMillis() - start));
|
||||
// Looks like delete doesn't work.
|
||||
assertTrue("Count is off: " + this.filter.count(),
|
||||
this.filter.count() == MAX_COUNT);
|
||||
}
|
||||
|
||||
public void testNote() {
|
||||
this.filter.note(this.getUri());
|
||||
assertFalse("Receiver was called", this.received);
|
||||
}
|
||||
|
||||
public void testForget() throws URIException {
|
||||
this.filter.forget(this.getUri(),
|
||||
new CrawlURI(UURIFactory.getInstance(this.getUri())));
|
||||
assertTrue("Didn't forget", this.filter.count() == 0);
|
||||
}
|
||||
|
||||
public void receive(CrawlURI item) {
|
||||
this.received = true;
|
||||
}
|
||||
|
||||
public String getUri() {
|
||||
return "http://www.archive.org";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user