diff --git a/engine/src/main/java/org/archive/crawler/framework/Scoper.java b/engine/src/main/java/org/archive/crawler/framework/Scoper.java index 3c8faf86..71ca2466 100644 --- a/engine/src/main/java/org/archive/crawler/framework/Scoper.java +++ b/engine/src/main/java/org/archive/crawler/framework/Scoper.java @@ -1,25 +1,22 @@ -/* Scoper - * - * Created on Jun 6, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + package org.archive.crawler.framework; import java.util.logging.FileHandler; @@ -80,7 +77,7 @@ public abstract class Scoper extends Processor implements Lifecycle { return this.scope; } @Autowired - public void setDecideRule(DecideRule scope) { + public void setScope(DecideRule scope) { this.scope = scope; } diff --git a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java index ccba9ca7..c03dc2cd 100644 --- a/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java +++ b/engine/src/main/java/org/archive/crawler/frontier/WorkQueue.java @@ -1,5 +1,24 @@ -package org.archive.crawler.frontier; +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.archive.crawler.frontier; + import java.io.IOException; import java.io.PrintWriter; import java.io.Serializable; @@ -14,7 +33,6 @@ import org.archive.crawler.framework.Frontier; import org.archive.crawler.frontier.precedence.PrecedenceProvider; import org.archive.crawler.frontier.precedence.SimplePrecedenceProvider; import org.archive.modules.CrawlURI; -import org.archive.modules.ProcessorURI; import org.archive.modules.fetcher.FetchStats; import org.archive.modules.fetcher.FetchStats.Stage; import org.archive.util.ArchiveUtils; @@ -665,9 +683,9 @@ public abstract class WorkQueue implements Frontier.FrontierGroup, } /* (non-Javadoc) - * @see org.archive.modules.fetcher.FetchStats.HasFetchStats#tally(org.archive.modules.ProcessorURI, org.archive.modules.fetcher.FetchStats.Stage) + * @see org.archive.modules.fetcher.FetchStats.HasFetchStats#tally(org.archive.modules.CrawlURI, org.archive.modules.fetcher.FetchStats.Stage) */ - public void tally(ProcessorURI curi, Stage stage) { + public void tally(CrawlURI curi, Stage stage) { substats.tally(curi, stage); precedenceProvider.tally(curi, stage); } diff --git a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java index d4baacf8..9546a82d 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/QuotaEnforcer.java @@ -1,24 +1,20 @@ -/* QuotaEnforcer - * - * Created on Nov 4, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.crawler.prefetch; @@ -31,7 +27,6 @@ import org.archive.crawler.framework.Frontier; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import org.archive.modules.fetcher.FetchStats; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.CrawlServer; @@ -310,15 +305,15 @@ public class QuotaEnforcer extends Processor { this.frontier = frontier; } - protected boolean shouldProcess(ProcessorURI puri) { + protected boolean shouldProcess(CrawlURI puri) { return puri instanceof CrawlURI; } - protected void innerProcess(ProcessorURI puri) { + protected void innerProcess(CrawlURI puri) { throw new AssertionError(); } - protected ProcessResult innerProcessResult(ProcessorURI puri) { + protected ProcessResult innerProcessResult(CrawlURI puri) { CrawlURI curi = (CrawlURI)puri; final CrawlServer server = ServerCacheUtil.getServerFor(serverCache, curi.getUURI()); diff --git a/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java index a5b55058..9e356e87 100644 --- a/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java +++ b/engine/src/main/java/org/archive/crawler/prefetch/RuntimeLimitEnforcer.java @@ -1,41 +1,35 @@ -/* RuntimeLimitEnforcer - * - * Created on July 7, 2006 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.crawler.prefetch; +import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_RUNTIME_LIMIT; + import java.util.logging.Level; import java.util.logging.Logger; - -import static org.archive.modules.fetcher.FetchStatusCodes.*; - import org.archive.crawler.framework.CrawlController; import org.archive.crawler.framework.CrawlStatus; import org.archive.crawler.reporting.StatisticsTracker; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import org.springframework.beans.factory.annotation.Autowired; @@ -160,18 +154,18 @@ public class RuntimeLimitEnforcer extends Processor { @Override - protected boolean shouldProcess(ProcessorURI puri) { + protected boolean shouldProcess(CrawlURI puri) { return puri instanceof CrawlURI; } @Override - protected void innerProcess(ProcessorURI curi) { + protected void innerProcess(CrawlURI curi) { throw new AssertionError(); } @Override - protected ProcessResult innerProcessResult(ProcessorURI curi) + protected ProcessResult innerProcessResult(CrawlURI curi) throws InterruptedException { CrawlController controller = getCrawlController(); StatisticsTracker stats = getStatisticsTracker(); diff --git a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java index 67e8c3c9..4065d7dd 100644 --- a/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java +++ b/engine/src/main/java/org/archive/crawler/processor/CrawlMapper.java @@ -1,25 +1,22 @@ -/* CrawlMapper - * - * Created on Sep 30, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + package org.archive.crawler.processor; import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR; @@ -35,7 +32,6 @@ import java.util.Iterator; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import org.archive.modules.deciderules.AcceptDecideRule; import org.archive.modules.deciderules.DecideResult; import org.archive.modules.deciderules.DecideRule; @@ -182,17 +178,17 @@ public abstract class CrawlMapper extends Processor implements Lifecycle { @Override - protected boolean shouldProcess(ProcessorURI puri) { + protected boolean shouldProcess(CrawlURI puri) { return true; } @Override - protected void innerProcess(ProcessorURI puri) { + protected void innerProcess(CrawlURI puri) { throw new AssertionError(); } @Override - protected ProcessResult innerProcessResult(ProcessorURI puri) { + protected ProcessResult innerProcessResult(CrawlURI puri) { CrawlURI curi = (CrawlURI)puri; String nowGeneration = ArchiveUtils.get14DigitDate().substring( diff --git a/engine/src/test/java/org/archive/crawler/postprocessor/CrawlStateUpdaterTest.java b/engine/src/test/java/org/archive/crawler/postprocessor/CrawlStateUpdaterTest.java deleted file mode 100644 index caf207f9..00000000 --- a/engine/src/test/java/org/archive/crawler/postprocessor/CrawlStateUpdaterTest.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2007 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * CrawlStateUpdater.java - * - * Created on Jan 31, 2007 - * - * $Id:$ - */ -package org.archive.crawler.postprocessor; - - -import org.archive.crawler.framework.CrawlerProcessorTestBase; - - -/** - * Unit test for {@link CrawlStateUpdater}. - * - * @author pjack - */ -public class CrawlStateUpdaterTest extends CrawlerProcessorTestBase { - - // TODO TESTME! - -} diff --git a/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java b/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java index 9cacadc6..7157a889 100644 --- a/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java +++ b/engine/src/test/java/org/archive/crawler/selftest/KeyWordProcessor.java @@ -1,10 +1,29 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.archive.crawler.selftest; import java.util.regex.Pattern; +import org.archive.io.ReplayCharSequence; import org.archive.modules.CrawlURI; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; /** @@ -67,26 +86,24 @@ public class KeyWordProcessor extends Processor { } @Override - protected void innerProcess(ProcessorURI uri) throws InterruptedException { - CrawlURI curi = (CrawlURI)uri; + protected void innerProcess(CrawlURI curi) throws InterruptedException { try { - CharSequence seq = uri.getRecorder().getReplayCharSequence(); - int precedence; - if (getPattern().matcher(seq).find()) { - precedence = getFoundPrecedence(); - } else { - precedence = getNotFoundPrecedence(); - } - for (CrawlURI c: curi.getOutCandidates()) { - c.setPrecedence(precedence); + CrawlURI viaUri = curi.getFullVia(); + if(!viaUri.getData().containsKey("keywordHit")) { + ReplayCharSequence seq = viaUri.getRecorder().getReplayCharSequence(); + viaUri.getData().put("keywordHit", getPattern().matcher(seq).find()); + seq.close(); } + boolean keywordHit = (Boolean) viaUri.getData().get("keywordHit"); + int precedence = keywordHit ? getFoundPrecedence() : getNotFoundPrecedence(); + curi.setPrecedence(precedence); } catch (Exception e) { e.printStackTrace(); } } @Override - protected boolean shouldProcess(ProcessorURI uri) { + protected boolean shouldProcess(CrawlURI uri) { if (!uri.getContentType().equals("text/html")) { return false; } diff --git a/modules/src/main/java/org/archive/modules/DefaultProcessorURI.java b/modules/src/main/java/org/archive/modules/DefaultProcessorURI.java deleted file mode 100644 index ab666220..00000000 --- a/modules/src/main/java/org/archive/modules/DefaultProcessorURI.java +++ /dev/null @@ -1,418 +0,0 @@ -package org.archive.modules; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.httpclient.HttpMethod; -import org.apache.commons.httpclient.URIException; -import org.archive.modules.credential.CredentialAvatar; -import org.archive.modules.extractor.Link; -import org.archive.modules.extractor.LinkContext; -import org.archive.modules.net.CrawlHost; -import org.archive.modules.net.CrawlServer; -import org.archive.modules.net.RobotsHonoringPolicy; -import org.archive.net.UURI; -import org.archive.util.Recorder; - - -public class DefaultProcessorURI -implements ProcessorURI { - - - /** - * An exception and an error message. Used for keeping list of errors. - */ - public static class ExceptionHolder { - - /** The exception. */ - final public Throwable exception; - - /** The error message. */ - final public String description; - - /** - * Constructor. - * - * @param e the exception - * @param desc the description - */ - public ExceptionHolder(Throwable e, String desc) { - this.exception = e; - this.description = desc; - } - } - - - final private Collection annotations = new ArrayList(); - final private Set avatars = - new HashSet(); - final private List nonFatalFailures = - new ArrayList(); - final private List outLinks = new ArrayList(); - final private List uriErrors - = new ArrayList(); - - private int discardedOutLinks; - - private String from; - private String userAgent; - - private long length; - private long size; - private String contentType; - - private long startTime; - private long endTime; - - private int fetchStatus; - private FetchType fetchType = FetchType.UNKNOWN; - private HttpMethod httpMethod; - - private String pathFromSeed; - - private UURI base; - - private Map data = new HashMap(); - - private Recorder recorder; - - private String resolvedName; - private RobotsHonoringPolicy robotsHonoringPolicy; - - private UURI uuri; - private UURI via; - private LinkContext viaContext; - - private boolean linkExtractionFinished; - private boolean location; - private boolean prereq; - private boolean seed; - - private boolean forceFetch; - - - public DefaultProcessorURI(UURI uuri, UURI via, LinkContext context) { - this.uuri = uuri; - this.via = via; - this.viaContext = context; - } - - public DefaultProcessorURI(UURI uuri, LinkContext context) { - this(uuri, null, context); - } - - public void addUriError(URIException e, String uri) { - uriErrors.add(new ExceptionHolder(e, uri)); - } - - public List getUriErrors() { - return uriErrors; - } - - public boolean attachRfc2617Credential(String realm) { - // TODO Auto-generated method stub - return false; - } - - public boolean detachRfc2617Credential(String realm) { - // TODO Auto-generated method stub - return false; - } - - public Collection getAnnotations() { - return annotations; - } - - public UURI getBaseURI() { - return base != null ? base : getUURI(); - } - - public long getContentLength() { - return length; - } - - - public void setContentLength(long len) { - this.length = len; - } - - public long getContentSize() { - return size; - } - - public String getContentType() { - return contentType; - } - - public CrawlHost getCrawlHost() { - // TODO Auto-generated method stub - return null; - } - - - public CrawlServer getCrawlServer(String serverKey) { - // TODO - return null; - } - - - public boolean hasCredentialAvatars() { - return getCredentialAvatars().size() > 0; - } - - - public Set getCredentialAvatars() { - return avatars; - } - - public String getDNSServerIPLabel() { - // TODO Auto-generated method stub - return null; - } - - public Map getData() { - return data; - } - - public long getFetchBeginTime() { - return startTime; - } - - public long getFetchCompletedTime() { - return endTime; - } - - public int getFetchStatus() { - return fetchStatus; - } - - public FetchType getFetchType() { - return fetchType; - } - - public String getFrom() { - return from; - } - - public Collection getNonFatalFailures() { - return nonFatalFailures; - } - - public List getOutLinks() { - return outLinks; - } - - public String getPathFromSeed() { - return pathFromSeed; - } - - public Recorder getRecorder() { - return recorder; - } - - public void setRecorder(Recorder r) { - this.recorder = r; - } - - public String getResolvedName() { - return resolvedName; // FIXME: CrawlHost - } - - public RobotsHonoringPolicy getRobotsHonoringPolicy() { - return robotsHonoringPolicy; - } - - public UURI getUURI() { - return uuri; - } - - public String getUserAgent() { - return userAgent; - } - - public void setUserAgent(String ua) { - this.userAgent = ua; - } - - public UURI getVia() { - return via; - } - - public LinkContext getViaContext() { - return viaContext; - } - - public boolean hasBeenLinkExtracted() { - return linkExtractionFinished; - } - - public boolean isLocation() { - return location; - } - - public boolean isPrerequisite() { - return prereq; - } - - public boolean isSeed() { - return seed; - } - - public boolean passedDNS() { - // TODO Auto-generated method stub - return false; - } - - public boolean populateCredentials(HttpMethod method) { - // TODO Auto-generated method stub - return false; - } - - public void promoteCredentials() { - // TODO Auto-generated method stub - - } - - public void requestCrawlPause() { - // TODO Auto-generated method stub - - } - - public void setBaseURI(UURI base) { - this.base = base; - } - - public void setContentDigest(String algorithm, byte[] digest) { - // TODO Auto-generated method stub - - } - - public void setContentSize(long size) { - this.size = size; - } - - public void setContentType(String mimeType) { - this.contentType = mimeType; - } - - public void setDNSServerIPLabel(String label) { - // TODO Auto-generated method stub - - } - - public void setError(String msg) { - // TODO Auto-generated method stub - } - - public void setFetchBeginTime(long time) { - startTime = time; - } - - public void setFetchCompletedTime(long time) { - endTime = time; - } - - public void setFetchStatus(int status) { - this.fetchStatus = status; - } - - public void setFetchType(FetchType type) { - if (type == null) { - throw new IllegalArgumentException("fetchType is non-null"); - } - this.fetchType = type; - } - - public void setHttpMethod(HttpMethod method) { - this.httpMethod = method; - } - - public void linkExtractorFinished() { - linkExtractionFinished = true; - } - - public void setPrerequisite(boolean prereq) { - this.prereq = prereq; - } - - public void setSeed(boolean seed) { - this.seed = seed; - } - - public void skipToPostProcessing() { - // TODO Auto-generated method stub - - } - - - public HttpMethod getHttpMethod() { - return this.httpMethod; - } - - - public int getFetchAttempts() { - return 1; // FIXME - } - - - public boolean containsDataKey(String k) { - return data.containsKey(k); - } - - - public byte[] getContentDigest() { - return null; // FIXME - } - - - public String getContentDigestSchemeString() { - // FIXME - return null; - } - - public void makeHeritable(String attr) { - // FIXME? -- maybe irrelevant - - } - - public boolean isSuccess() { - // FIXME? - return false; - } - - public Map getPersistentDataMap() { - // FIXME? - return null; - } - - public void addPersistentDataMapKey(String s) { - - } - - - public void incrementDiscardedOutLinks() { - discardedOutLinks++; - } - - - public boolean forceFetch() { - return forceFetch; - } - - public void setForceFetch(boolean b) { - this.forceFetch = b; - } - - public String getSourceTag() { - return ""; - } - - public String getURI() { - return uuri.toCustomString(); - } -} diff --git a/modules/src/main/java/org/archive/modules/ProcessorURI.java b/modules/src/main/java/org/archive/modules/ProcessorURI.java deleted file mode 100644 index 2fc1d748..00000000 --- a/modules/src/main/java/org/archive/modules/ProcessorURI.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules; - - -import java.util.Collection; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.httpclient.HttpMethod; -import org.archive.modules.credential.CredentialAvatar; -import org.archive.modules.extractor.Link; -import org.archive.modules.extractor.LinkContext; -import org.archive.net.UURI; -import org.archive.util.Recorder; - - -/** - * A URI to be processed. - * - * @author pjack - */ -public interface ProcessorURI { - - - public static enum FetchType { HTTP_GET, HTTP_POST, UNKNOWN }; - - /** - * Returns the URI being processed. - * - * @return the URI - */ - UURI getUURI(); - - public String getURI(); - - UURI getVia(); - - boolean isPrerequisite(); - void setPrerequisite(boolean prereq); - - // Used to be a map attribute. - void setError(String msg); - - // Used to be a map attribute. - long getFetchBeginTime(); - void setFetchBeginTime(long time); - - // Used to be a map attribute - long getFetchCompletedTime(); - void setFetchCompletedTime(long time); - - String getDNSServerIPLabel(); - void setDNSServerIPLabel(String label); - - FetchType getFetchType(); - void setFetchType(FetchType type); - - Recorder getRecorder(); - - boolean containsDataKey(String attr); - Map getData(); - void makeHeritable(String attr); - Map getPersistentDataMap(); - void addPersistentDataMapKey(String s); - - String getUserAgent(); - void setUserAgent(String ua); - - long getContentSize(); - void setContentSize(long size); - - String getContentDigestSchemeString(); - byte[] getContentDigest(); - void setContentDigest(String algorithm, byte[] digest); - - String getContentType(); - void setContentType(String mimeType); - - long getContentLength(); - - Collection getAnnotations(); - Collection getNonFatalFailures(); - - int getFetchStatus(); - void setFetchStatus(int status); - - // Used to be a map attribute. May still want to be one. - HttpMethod getHttpMethod(); - void setHttpMethod(HttpMethod method); - - - boolean hasCredentialAvatars(); - Set getCredentialAvatars(); - - String getPathFromSeed(); - boolean isSeed(); - void setSeed(boolean seed); - - boolean isLocation(); - - - Collection getOutLinks(); - - UURI getBaseURI(); - void setBaseURI(UURI base); - - boolean hasBeenLinkExtracted(); - void linkExtractorFinished(); - - LinkContext getViaContext(); - - int getFetchAttempts(); - - boolean isSuccess(); - - void incrementDiscardedOutLinks(); - - String getSourceTag(); - - boolean forceFetch(); - -} diff --git a/modules/src/main/java/org/archive/modules/credential/Credential.java b/modules/src/main/java/org/archive/modules/credential/Credential.java index d612f78d..9f8d95de 100644 --- a/modules/src/main/java/org/archive/modules/credential/Credential.java +++ b/modules/src/main/java/org/archive/modules/credential/Credential.java @@ -1,24 +1,20 @@ -/* Credential +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Created on Apr 1, 2004 + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Copyright (C) 2004 Internet Archive. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * http://www.apache.org/licenses/LICENSE-2.0 * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.credential; @@ -31,7 +27,7 @@ import javax.management.AttributeNotFoundException; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlServer; import org.archive.modules.net.ServerCache; import org.archive.modules.net.ServerCacheUtil; @@ -95,21 +91,21 @@ public abstract class Credential implements Serializable { * * Override if credential knows internally what it wants to attach as * payload. Otherwise, if payload is external, use the below - * {@link #attach(ProcessorURI, String)}. + * {@link #attach(CrawlURI, String)}. * - * @param curi ProcessorURI to load with credentials. + * @param curi CrawlURI to load with credentials. */ - public void attach(ProcessorURI curi) { + public void attach(CrawlURI curi) { attach(curi, null); } /** * Attach this credentials avatar to the passed curi . * - * @param curi ProcessorURI to load with credentials. + * @param curi CrawlURI to load with credentials. * @param payload Payload to carry in avatar. Usually credentials. */ - public void attach(ProcessorURI curi, String payload) { + public void attach(CrawlURI curi, String payload) { CredentialAvatar ca = (payload == null )? new CredentialAvatar(this.getClass(), getKey()): new CredentialAvatar(this.getClass(), getKey(), payload); @@ -122,7 +118,7 @@ public abstract class Credential implements Serializable { * @param curi * @return True if we detached a Credential reference. */ - public boolean detach(ProcessorURI curi) { + public boolean detach(CrawlURI curi) { boolean result = false; Set avatars = curi.getCredentialAvatars(); if (avatars.isEmpty()) { @@ -147,7 +143,7 @@ public abstract class Credential implements Serializable { * @param curi * @return True if we detached references. */ - public boolean detachAll(ProcessorURI curi) { + public boolean detachAll(CrawlURI curi) { boolean result = false; Set avatars = curi.getCredentialAvatars(); if (avatars.isEmpty()) { @@ -166,26 +162,26 @@ public abstract class Credential implements Serializable { } /** - * @param curi ProcessorURI to look at. + * @param curi CrawlURI to look at. * @return True if this credential IS a prerequisite for passed - * ProcessorURI. + * CrawlURI. */ - public abstract boolean isPrerequisite(ProcessorURI curi); + public abstract boolean isPrerequisite(CrawlURI curi); /** - * @param curi ProcessorURI to look at. - * @return True if this credential HAS a prerequisite for passed ProcessorURI. + * @param curi CrawlURI to look at. + * @return True if this credential HAS a prerequisite for passed CrawlURI. */ - public abstract boolean hasPrerequisite(ProcessorURI curi); + public abstract boolean hasPrerequisite(CrawlURI curi); /** * Return the authentication URI, either absolute or relative, that serves * as prerequisite the passed curi. * - * @param curi ProcessorURI to look at. + * @param curi CrawlURI to look at. * @return Prerequisite URI for the passed curi. */ - public abstract String getPrerequisite(ProcessorURI curi); + public abstract String getPrerequisite(CrawlURI curi); /** * @param context Context to use when searching for credential domain. @@ -202,17 +198,17 @@ public abstract class Credential implements Serializable { public abstract boolean isEveryTime(); /** - * @param curi ProcessorURI to as for context. + * @param curi CrawlURI to as for context. * @param http Instance of httpclient. * @param method Method to populate. * @param payload Avatar payload to use populating the method. * @return True if added a credentials. */ - public abstract boolean populate(ProcessorURI curi, HttpClient http, + public abstract boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, String payload); /** - * @param curi ProcessorURI to look at. + * @param curi CrawlURI to look at. * @return True if this credential is to be posted. Return false if the * credential is to be GET'd or if POST'd or GET'd are not pretinent to this * credential type. @@ -222,11 +218,11 @@ public abstract class Credential implements Serializable { /** * Test passed curi matches this credentials rootUri. * @param controller - * @param curi ProcessorURI to test. + * @param curi CrawlURI to test. * @return True if domain for credential matches that of the passed curi. */ public boolean rootUriMatch(ServerCache cache, - ProcessorURI curi) { + CrawlURI curi) { String cd = getDomain(); CrawlServer serv = ServerCacheUtil.getServerFor(cache, curi.getUURI()); diff --git a/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java b/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java index 6dcd5e56..1f4c8190 100644 --- a/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java +++ b/modules/src/main/java/org/archive/modules/credential/CredentialAvatar.java @@ -25,7 +25,7 @@ import java.util.Iterator; import java.util.List; import java.util.logging.Logger; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * A credential representation. @@ -184,7 +184,7 @@ implements Serializable { * @param curi ProcessorURI to use for context. * @return The credential this avatar represents. */ - public Credential getCredential(CredentialStore cs, ProcessorURI curi) { + public Credential getCredential(CredentialStore cs, CrawlURI curi) { Credential result = null; if (cs == null) { diff --git a/modules/src/main/java/org/archive/modules/credential/CredentialStore.java b/modules/src/main/java/org/archive/modules/credential/CredentialStore.java index a84c89bd..336397bb 100644 --- a/modules/src/main/java/org/archive/modules/credential/CredentialStore.java +++ b/modules/src/main/java/org/archive/modules/credential/CredentialStore.java @@ -34,7 +34,7 @@ import javax.management.AttributeNotFoundException; import javax.management.MBeanException; import javax.management.ReflectionException; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.spring.HasKeyedProperties; import org.archive.spring.KeyedProperties; @@ -140,7 +140,7 @@ public class CredentialStore implements Serializable, HasKeyedProperties { * credentials. * @return Unmodifable sublist of all elements of passed type. */ - public Set subset(ProcessorURI context, Class type) { + public Set subset(CrawlURI context, Class type) { return subset(context, type, null); } @@ -157,7 +157,7 @@ public class CredentialStore implements Serializable, HasKeyedProperties { * but different ports (e.g. http and https). * @return Unmodifable sublist of all elements of passed type. */ - public Set subset(ProcessorURI context, Class type, String rootUri) { + public Set subset(CrawlURI context, Class type, String rootUri) { Set result = null; for (Credential c: getAll()) { if (!type.isInstance(c)) { @@ -181,7 +181,7 @@ public class CredentialStore implements Serializable, HasKeyedProperties { } - public Credential getCredential(ProcessorURI curi, CredentialAvatar ca) { + public Credential getCredential(CrawlURI curi, CredentialAvatar ca) { Credential result = null; Collection all = getAll(); diff --git a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java index d8197149..5ce10b87 100644 --- a/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HtmlFormCredential.java @@ -32,7 +32,7 @@ import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.lang.StringUtils; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -93,7 +93,7 @@ public class HtmlFormCredential extends Credential { public HtmlFormCredential() { } - public boolean isPrerequisite(final ProcessorURI curi) { + public boolean isPrerequisite(final CrawlURI curi) { boolean result = false; String curiStr = curi.getUURI().toString(); String loginUri = getPrerequisite(curi); @@ -116,11 +116,11 @@ public class HtmlFormCredential extends Credential { return result; } - public boolean hasPrerequisite(ProcessorURI curi) { + public boolean hasPrerequisite(CrawlURI curi) { return getPrerequisite(curi) != null; } - public String getPrerequisite(ProcessorURI curi) { + public String getPrerequisite(CrawlURI curi) { return getLoginUri(); } @@ -133,7 +133,7 @@ public class HtmlFormCredential extends Credential { return false; } - public boolean populate(ProcessorURI curi, HttpClient http, HttpMethod method, + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, String payload) { // http is not used. // payload is not used. diff --git a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java index 1cceddfb..e6afdbb0 100644 --- a/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java +++ b/modules/src/main/java/org/archive/modules/credential/HttpAuthenticationCredential.java @@ -28,7 +28,7 @@ import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthScope; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * A Basic/Digest HTTP Authentication (RFC2617) credential. @@ -78,19 +78,19 @@ public class HttpAuthenticationCredential extends Credential { public HttpAuthenticationCredential() { } - public boolean isPrerequisite(ProcessorURI curi) { + public boolean isPrerequisite(CrawlURI curi) { // Return false. Later when we implement preemptive // rfc2617, this will change. return false; } - public boolean hasPrerequisite(ProcessorURI curi) { + public boolean hasPrerequisite(CrawlURI curi) { // Return false. Later when we implement preemptive // rfc2617, this will change. return false; } - public String getPrerequisite(ProcessorURI curi) { + public String getPrerequisite(CrawlURI curi) { // Return null. Later when we implement preemptive // rfc2617, this will change. return null; @@ -104,7 +104,7 @@ public class HttpAuthenticationCredential extends Credential { return true; } - public boolean populate(ProcessorURI curi, HttpClient http, HttpMethod method, + public boolean populate(CrawlURI curi, HttpClient http, HttpMethod method, String payload) { boolean result = false; String authRealm = payload; @@ -161,7 +161,7 @@ public class HttpAuthenticationCredential extends Credential { * found. */ public static HttpAuthenticationCredential getByRealm(Set rfc2617Credentials, - String realm, ProcessorURI context) { + String realm, CrawlURI context) { HttpAuthenticationCredential result = null; if (rfc2617Credentials == null || rfc2617Credentials.size() <= 0) { diff --git a/modules/src/main/java/org/archive/modules/deciderules/AcceptDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/AcceptDecideRule.java index 06ab7f9d..e139e319 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/AcceptDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/AcceptDecideRule.java @@ -1,29 +1,24 @@ -/* Copyright (C) 2006 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * AcceptDecideRule.java - * Created on October 5, 2006 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; public class AcceptDecideRule extends DecideRule { @@ -31,13 +26,13 @@ public class AcceptDecideRule extends DecideRule { @Override - protected DecideResult innerDecide(ProcessorURI uri) { + protected DecideResult innerDecide(CrawlURI uri) { return DecideResult.ACCEPT; } @Override - public DecideResult onlyDecision(ProcessorURI uri) { + public DecideResult onlyDecision(CrawlURI uri) { return DecideResult.ACCEPT; } diff --git a/modules/src/main/java/org/archive/modules/deciderules/AddRedirectFromRootServerToScope.java b/modules/src/main/java/org/archive/modules/deciderules/AddRedirectFromRootServerToScope.java index 19595007..1f17523a 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/AddRedirectFromRootServerToScope.java +++ b/modules/src/main/java/org/archive/modules/deciderules/AddRedirectFromRootServerToScope.java @@ -1,30 +1,26 @@ -/* AddRedirectFromRootServerToScope - * - * Created on May 25, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; @@ -41,7 +37,7 @@ extends PredicatedDecideRule { } @Override - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { UURI via = uri.getVia(); if (via == null) { return false; diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentLengthDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentLengthDecideRule.java index e26340e8..ce8f02e1 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ContentLengthDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ContentLengthDecideRule.java @@ -1,28 +1,24 @@ -/* $Id$ - * - * Created on 28.8.2006 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2006 Olaf Freyer - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; public class ContentLengthDecideRule extends DecideRule { @@ -51,7 +47,7 @@ public class ContentLengthDecideRule extends DecideRule { } - protected DecideResult innerDecide(ProcessorURI uri) { + protected DecideResult innerDecide(CrawlURI uri) { if (uri.getContentLength() < getContentLengthThreshold()) { return DecideResult.ACCEPT; } diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java index 48d8da36..8f910800 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeMatchesRegExpDecideRule.java @@ -1,26 +1,24 @@ -/* $Id: $ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2007 Olaf Freyer + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -37,7 +35,7 @@ public class ContentTypeMatchesRegExpDecideRule extends MatchesRegExpDecideRule{ @Override - protected String getString(ProcessorURI uri) { + protected String getString(CrawlURI uri) { return uri.getContentType(); } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java index 20997ddc..039ecace 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ContentTypeNotMatchesRegExpDecideRule.java @@ -1,27 +1,24 @@ - -/* $Id: $ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2007 Olaf Freyer + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * DecideRule whose decision is applied if the URI's content-type @@ -44,7 +41,7 @@ public class ContentTypeNotMatchesRegExpDecideRule extends * @return true if the regexp is not matched */ @Override - protected boolean evaluate(ProcessorURI o) { + protected boolean evaluate(CrawlURI o) { return !super.evaluate(o); } diff --git a/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java index 4fa430d3..e6086cff 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/DecideRule.java @@ -1,32 +1,27 @@ -/* Copyright (C) 2006 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * DecideRule.java - * Created on October 5, 2006 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; import java.io.Serializable; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.spring.HasKeyedProperties; import org.archive.spring.KeyedProperties; @@ -58,7 +53,7 @@ public abstract class DecideRule implements Serializable, HasKeyedProperties { } - public DecideResult decisionFor(ProcessorURI uri) { + public DecideResult decisionFor(CrawlURI uri) { if (!getEnabled()) { return DecideResult.NONE; } @@ -71,14 +66,14 @@ public abstract class DecideRule implements Serializable, HasKeyedProperties { } - protected abstract DecideResult innerDecide(ProcessorURI uri); + protected abstract DecideResult innerDecide(CrawlURI uri); - public DecideResult onlyDecision(ProcessorURI uri) { + public DecideResult onlyDecision(CrawlURI uri) { return null; } - public boolean accepts(ProcessorURI uri) { + public boolean accepts(CrawlURI uri) { return DecideResult.ACCEPT == decisionFor(uri); } diff --git a/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java b/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java index 0b17b07a..be0f2c06 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java +++ b/modules/src/main/java/org/archive/modules/deciderules/DecideRuleSequence.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; public class DecideRuleSequence extends DecideRule { final private static Logger LOGGER = @@ -38,7 +38,7 @@ public class DecideRuleSequence extends DecideRule { kp.put("rules", rules); } - public DecideResult innerDecide(ProcessorURI uri) { + public DecideResult innerDecide(CrawlURI uri) { DecideResult result = DecideResult.NONE; List rules = getRules(); int max = rules.size(); diff --git a/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java index 7b174ef7..3d1fc0a3 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ExternalGeoLocationDecideRule.java @@ -1,24 +1,20 @@ -/* ExternalGeoLocationDecideRule - * - * Created on May 25, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; @@ -31,7 +27,7 @@ import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; import org.springframework.beans.factory.annotation.Autowired; @@ -91,7 +87,7 @@ public class ExternalGeoLocationDecideRule extends PredicatedDecideRule { } @Override - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { ExternalGeoLookupInterface impl = getLookup(); if (impl == null) { return false; diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java index 6661fa93..674f5d26 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusDecideRule.java @@ -1,33 +1,27 @@ -/* FetchStatusDecideRule -* -* $Id$ -* -* Created on Aug 11, 2006 -* -* Copyright (C) 2006 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; import java.util.ArrayList; import java.util.List; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Rule applies the configured decision for any URI which has a @@ -62,7 +56,7 @@ public class FetchStatusDecideRule extends PredicatedDecideRule { * Evaluate whether given object is equal to the configured status */ @Override - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { return getStatusCodes().contains(uri.getFetchStatus()); } diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java index f3a4e7b2..2e96044a 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusMatchesRegExpDecideRule.java @@ -1,29 +1,25 @@ -/* $Id$ -* -* Created on Sep 4, 2006 -* -* Copyright (C) 2006 Olaf Freyer. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; @@ -39,7 +35,7 @@ public class FetchStatusMatchesRegExpDecideRule extends MatchesRegExpDecideRule } - protected String getString(ProcessorURI uri) { + protected String getString(CrawlURI uri) { return Integer.toString(uri.getFetchStatus()); } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java index 8ffc8103..518d843d 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/FetchStatusNotMatchesRegExpDecideRule.java @@ -1,28 +1,24 @@ -/* $Id: FetchStatusNotMatchesRegExpDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ -* -* Created on Sep 4, 2006 -* -* Copyright (C) 2006 Olaf Freyer. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; public class FetchStatusNotMatchesRegExpDecideRule @@ -46,7 +42,7 @@ extends FetchStatusMatchesRegExpDecideRule { * @return true if the regexp is not matched */ @Override - protected boolean evaluate(ProcessorURI object) { + protected boolean evaluate(CrawlURI object) { return ! super.evaluate(object); } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/deciderules/HasViaDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/HasViaDecideRule.java index 1542e549..c71d6fcd 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/HasViaDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/HasViaDecideRule.java @@ -1,30 +1,24 @@ -/* HasViaDecideRule -* -* $Id$ -* -* Created on Aug 11, 2006 -* -* Copyright (C) 2006 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Rule applies the configured decision for any URI which has a 'via' @@ -48,7 +42,7 @@ public class HasViaDecideRule extends PredicatedDecideRule { * hops. */ @Override - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { return uri.getVia() != null; } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/HopCrossesAssignmentLevelDomainDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/HopCrossesAssignmentLevelDomainDecideRule.java index e9fb4680..e174f5db 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/HopCrossesAssignmentLevelDomainDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/HopCrossesAssignmentLevelDomainDecideRule.java @@ -21,7 +21,7 @@ package org.archive.modules.deciderules; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.PublicSuffixes; import org.archive.net.UURI; @@ -42,7 +42,7 @@ public class HopCrossesAssignmentLevelDomainDecideRule extends PredicatedDecideR public HopCrossesAssignmentLevelDomainDecideRule() { } - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { UURI via = uri.getVia(); if (via == null) { return false; diff --git a/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java index 2d672de0..3a0945b1 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/HopsPathMatchesRegExpDecideRule.java @@ -1,30 +1,24 @@ -/* HopsPathMatchesRegExpDecideRule -* -* $Id$ -* -* Created on June 23, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Rule applies configured decision to any CrawlURIs whose 'hops-path' @@ -44,7 +38,7 @@ public class HopsPathMatchesRegExpDecideRule extends MatchesRegExpDecideRule { @Override - protected String getString(ProcessorURI uri) { + protected String getString(CrawlURI uri) { return uri.getPathFromSeed(); } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java index 63ad2504..92a1690e 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/MatchesListRegExpDecideRule.java @@ -1,26 +1,20 @@ -/* MatchesListRegExpDecideRule - * - * $Id$ - * - * Created on 30.5.2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Kristinn Sigurdsson - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; @@ -30,7 +24,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Rule applies configured decision to any CrawlURIs whose String URI @@ -91,7 +85,7 @@ public class MatchesListRegExpDecideRule extends PredicatedDecideRule { * matches configured regexps */ @Override - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { List regexps = getRegexList(); if(regexps.size()==0){ return false; diff --git a/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java index 6dc4825d..1f8537ab 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/MatchesRegExpDecideRule.java @@ -1,35 +1,29 @@ -/* MatchesRegExpDecideRule -* -* $Id$ -* -* Created on Apr 4, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; import java.util.regex.Pattern; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** - * Rule applies configured decision to any ProcessorURIs whose String URI + * Rule applies configured decision to any CrawlURIs whose String URI * matches the supplied regexp. * * @author gojomo @@ -63,12 +57,12 @@ public class MatchesRegExpDecideRule extends PredicatedDecideRule { * @return true if regexp is matched */ @Override - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { Pattern p = getRegex(); return p.matcher(getString(uri)).matches(); } - protected String getString(ProcessorURI uri) { + protected String getString(CrawlURI uri) { return uri.toString(); } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotExceedsDocumentLengthTresholdDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotExceedsDocumentLengthTresholdDecideRule.java index bb02d809..0f56fe9f 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/NotExceedsDocumentLengthTresholdDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/NotExceedsDocumentLengthTresholdDecideRule.java @@ -1,24 +1,20 @@ -/* $Id: NotExceedsDocumentLengthTresholdDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ - * - * Created on 28.8.2006 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2006 Olaf Freyer - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; @@ -26,7 +22,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.httpclient.HttpMethod; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Applies configured decision for URIs with content length less than a given @@ -79,7 +75,7 @@ extends PredicatedDecideRule { public NotExceedsDocumentLengthTresholdDecideRule() { } - protected boolean evaluate(ProcessorURI curi) { + protected boolean evaluate(CrawlURI curi) { int contentlength = HEADER_PREDICTS_MISSING; // filter used as midfetch filter diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java index 9d553537..89a38390 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesFilePatternDecideRule.java @@ -1,30 +1,24 @@ -/* NotMatchesFilePatternDecideRule -* -* $Id: NotMatchesFilePatternDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ -* -* Created on Apr 4, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -55,7 +49,7 @@ extends MatchesFilePatternDecideRule { * @return true if the regexp is not matched */ @Override - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { return ! super.evaluate(uri); } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java index 4dd26a87..d9d4633c 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesListRegExpDecideRule.java @@ -1,30 +1,24 @@ -/* NotMatchesListRegExpDecideRule - * - * $Id: NotMatchesListRegExpDecideRule.java 4721 2006-11-14 20:03:18Z stack-sf $ - * - * Created on 30.5.2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2005 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -56,7 +50,7 @@ public class NotMatchesListRegExpDecideRule extends MatchesListRegExpDecideRule * @return true if the regexps are not matched */ @Override - protected boolean evaluate(ProcessorURI object) { + protected boolean evaluate(CrawlURI object) { return ! super.evaluate(object); } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java index 242ac87d..9b8e1252 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/NotMatchesRegExpDecideRule.java @@ -1,30 +1,24 @@ -/* NotMatchesRegExpDecideRule -* -* $Id: NotMatchesRegExpDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ -* -* Created on Apr 4, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -54,7 +48,7 @@ public class NotMatchesRegExpDecideRule extends MatchesRegExpDecideRule { * @return true if the regexp is not matched */ @Override - protected boolean evaluate(ProcessorURI object) { + protected boolean evaluate(CrawlURI object) { return ! super.evaluate(object); } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java index 726b5356..81e70b04 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/PathologicalPathDecideRule.java @@ -1,33 +1,27 @@ -/* PathologicalPathDecideRule -* -* $Id$ -* -* Created on Apr 1, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -67,7 +61,7 @@ public class PathologicalPathDecideRule extends DecideRule { @Override - protected DecideResult innerDecide(ProcessorURI uri) { + protected DecideResult innerDecide(CrawlURI uri) { int maxRep = getMaxRepetitions(); Pattern p = getPattern(maxRep); if (p.matcher(uri.getUURI().toString()).matches()) { diff --git a/modules/src/main/java/org/archive/modules/deciderules/PredicatedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/PredicatedDecideRule.java index 079930b0..ebdce650 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/PredicatedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/PredicatedDecideRule.java @@ -1,30 +1,24 @@ -/* AcceptRule -* -* $Id: PredicatedDecideRule.java 4914 2007-02-18 21:53:01Z gojomo $ -* -* Created on Mar 3, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Rule which applies the configured decision only if a @@ -49,12 +43,12 @@ public abstract class PredicatedDecideRule extends DecideRule { } @Override - protected DecideResult innerDecide(ProcessorURI uri) { + protected DecideResult innerDecide(CrawlURI uri) { if (evaluate(uri)) { return getDecision(); } return DecideResult.NONE; } - protected abstract boolean evaluate(ProcessorURI object); + protected abstract boolean evaluate(CrawlURI object); } diff --git a/modules/src/main/java/org/archive/modules/deciderules/PrerequisiteAcceptDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/PrerequisiteAcceptDecideRule.java index 988d3666..583a687d 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/PrerequisiteAcceptDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/PrerequisiteAcceptDecideRule.java @@ -1,30 +1,24 @@ -/* AcceptDecideRule -* -* $Id$ -* -* Created on Mar 3, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.extractor.Hop; @@ -43,7 +37,7 @@ public class PrerequisiteAcceptDecideRule extends DecideRule { } - public DecideResult innerDecide(ProcessorURI uri) { + public DecideResult innerDecide(CrawlURI uri) { String hopsPath = uri.getPathFromSeed(); if (hopsPath != null && hopsPath.length() > 0 && hopsPath.charAt(hopsPath.length()-1) == Hop.PREREQ.getHopChar()) { diff --git a/modules/src/main/java/org/archive/modules/deciderules/RejectDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/RejectDecideRule.java index 0978a8a6..83f4988e 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/RejectDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/RejectDecideRule.java @@ -1,29 +1,24 @@ -/* Copyright (C) 2006 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * RejectDecideRule.java - * Created on October 5, 2006 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; public class RejectDecideRule extends DecideRule { @@ -32,13 +27,13 @@ public class RejectDecideRule extends DecideRule { @Override - protected DecideResult innerDecide(ProcessorURI uri) { + protected DecideResult innerDecide(CrawlURI uri) { return DecideResult.REJECT; } @Override - public DecideResult onlyDecision(ProcessorURI uri) { + public DecideResult onlyDecision(CrawlURI uri) { return DecideResult.REJECT; } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java index 571d6626..76478f32 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/ScriptedDecideRule.java @@ -32,7 +32,7 @@ import javax.script.ScriptException; import org.apache.commons.io.IOUtils; import org.archive.io.ReadSource; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.springframework.beans.BeansException; import org.springframework.beans.factory.BeanInitializationException; import org.springframework.beans.factory.InitializingBean; @@ -121,7 +121,7 @@ implements ApplicationContextAware, InitializingBean { } } @Override - public DecideResult innerDecide(ProcessorURI uri) { + public DecideResult innerDecide(CrawlURI uri) { // depending on previous configuration, engine may // be local to this thread or shared ScriptEngine engine = getEngine(); diff --git a/modules/src/main/java/org/archive/modules/deciderules/SeedAcceptDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/SeedAcceptDecideRule.java index 5646c8e4..4a47a27f 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/SeedAcceptDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/SeedAcceptDecideRule.java @@ -1,30 +1,24 @@ -/* SeedAcceptDecideRule -* -* $Id$ -* -* Created on Sep 13, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -44,7 +38,7 @@ public class SeedAcceptDecideRule extends DecideRule { @Override - protected DecideResult innerDecide(ProcessorURI uri) { + protected DecideResult innerDecide(CrawlURI uri) { if (uri.isSeed()) { return DecideResult.ACCEPT; } diff --git a/modules/src/main/java/org/archive/modules/deciderules/TooManyHopsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/TooManyHopsDecideRule.java index 9363a983..4278825f 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/TooManyHopsDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/TooManyHopsDecideRule.java @@ -1,30 +1,24 @@ -/* AcceptRule -* -* $Id$ -* -* Created on Apr 1, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; @@ -71,7 +65,7 @@ public class TooManyHopsDecideRule extends PredicatedDecideRule { * @return true if the mx-hops is exceeded */ @Override - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { String hops = uri.getPathFromSeed(); if (hops == null) { return false; diff --git a/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java index 13764577..85268b2d 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/TooManyPathSegmentsDecideRule.java @@ -1,30 +1,24 @@ -/* AcceptRule -* -* $Id$ -* -* Created on Apr 1, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Rule REJECTs any CrawlURIs whose total number of path-segments (as @@ -68,7 +62,7 @@ public class TooManyPathSegmentsDecideRule extends PredicatedDecideRule { * @return true if the path-segments is exceeded */ @Override - protected boolean evaluate(ProcessorURI curi) { + protected boolean evaluate(CrawlURI curi) { String uri = curi.toString(); int count = 0; int threshold = getMaxPathDepth(); diff --git a/modules/src/main/java/org/archive/modules/deciderules/TransclusionDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/TransclusionDecideRule.java index 3eff2784..338c4886 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/TransclusionDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/TransclusionDecideRule.java @@ -1,30 +1,24 @@ -/* TransclusionDecideRule -* -* $Id$ -* -* Created on Apr 1, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.extractor.Hop; /** @@ -89,7 +83,7 @@ public class TransclusionDecideRule extends PredicatedDecideRule { * @param object Object to make decision on. * @return true if the transitive hops >0 and <= max */ - protected boolean evaluate(ProcessorURI curi) { + protected boolean evaluate(CrawlURI curi) { String hopsPath = curi.getPathFromSeed(); if (hopsPath == null || hopsPath.length() == 0) { return false; diff --git a/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java index d225b238..1d941200 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/recrawl/IdenticalDigestDecideRule.java @@ -24,7 +24,7 @@ import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HIST import java.util.Map; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideResult; import org.archive.modules.deciderules.PredicatedDecideRule; @@ -55,7 +55,7 @@ public class IdenticalDigestDecideRule extends PredicatedDecideRule { * @param object should be CrawlURI * @return true if current-fetch content-digest matches previous */ - protected boolean evaluate(ProcessorURI curi) { + protected boolean evaluate(CrawlURI curi) { return hasIdenticalDigest(curi); } @@ -70,7 +70,7 @@ public class IdenticalDigestDecideRule extends PredicatedDecideRule { * otherwise false */ @SuppressWarnings("unchecked") - public static boolean hasIdenticalDigest(ProcessorURI curi) { + public static boolean hasIdenticalDigest(CrawlURI curi) { if(curi.containsDataKey(A_FETCH_HISTORY)) { Map[] history = (Map[])curi.getData().get(A_FETCH_HISTORY); diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnDomainsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnDomainsDecideRule.java index be367945..029ac52b 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnDomainsDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnDomainsDecideRule.java @@ -24,7 +24,7 @@ */ package org.archive.modules.deciderules.surt; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -54,7 +54,7 @@ public class NotOnDomainsDecideRule extends OnDomainsDecideRule { * @param object to evaluate * @return true if URI is not in domain set */ - protected boolean evaluate(ProcessorURI object) { + protected boolean evaluate(CrawlURI object) { boolean superDecision = super.evaluate(object); return !superDecision; } diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnHostsDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnHostsDecideRule.java index 46969f7b..520a9d94 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnHostsDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/NotOnHostsDecideRule.java @@ -1,30 +1,24 @@ -/* NotOnHostsDecideRule -* -* $Id: NotOnHostsDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ -* -* Created on Apr 5, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules.surt; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -53,7 +47,7 @@ public class NotOnHostsDecideRule extends OnHostsDecideRule { * @return true if URI not in set */ @Override - protected boolean evaluate(ProcessorURI object) { + protected boolean evaluate(CrawlURI object) { boolean superDecision = super.evaluate(object); return !superDecision; } diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java index 0855bfb9..17e24ae0 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/NotSurtPrefixedDecideRule.java @@ -1,30 +1,24 @@ -/* NotSurtPrefixedDecideRule -* -* $Id: NotSurtPrefixedDecideRule.java 4649 2006-09-25 17:16:55Z paul_jack $ -* -* Created on Apr 5, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules.surt; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -57,7 +51,7 @@ public class NotSurtPrefixedDecideRule extends SurtPrefixedDecideRule { * @param object * @return true if regexp is matched */ - protected boolean evaluate(ProcessorURI object) { + protected boolean evaluate(CrawlURI object) { return !super.evaluate(object); } } diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java index 7a9b4bce..5fbd9b39 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java @@ -1,31 +1,27 @@ -/* ScopePlusOneDecideRule -* -* Created on Aug 22, 2005 -* -* Copyright 2005 Regents of the University of California, All rights reserved -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.deciderules.surt; import java.util.logging.Level; import java.util.logging.Logger; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideResult; import org.archive.net.UURI; import org.archive.util.SurtPrefixSet; @@ -70,7 +66,7 @@ public class ScopePlusOneDecideRule extends SurtPrefixedDecideRule { * @return true if URI is either in scope or its via is */ @Override - protected DecideResult innerDecide(ProcessorURI uri) { + protected DecideResult innerDecide(CrawlURI uri) { SurtPrefixSet set = getPrefixes(); UURI u = uri.getUURI(); // First, is the URI itself in scope? diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java index 50a692b2..94972258 100644 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java +++ b/modules/src/main/java/org/archive/modules/deciderules/surt/SurtPrefixedDecideRule.java @@ -28,7 +28,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.lang.StringUtils; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.deciderules.DecideResult; import org.archive.modules.deciderules.PredicatedDecideRule; import org.archive.modules.seeds.SeedListener; @@ -165,7 +165,7 @@ implements * @return true if item, as SURT form URI, is prefixed by an item in the set */ @Override - protected boolean evaluate(ProcessorURI uri) { + protected boolean evaluate(CrawlURI uri) { if (getAlsoCheckVia()) { if (innerDecide(uri.getVia())) { return true; @@ -244,9 +244,9 @@ implements /** * If appropriate, convert seed notification into prefix-addition. * - * @see org.archive.modules.seeds.SeedListener#addedSeed(org.archive.modules.ProcessorURI) + * @see org.archive.modules.seeds.SeedListener#addedSeed(org.archive.modules.CrawlURI) */ - public void addedSeed(final ProcessorURI curi) { + public void addedSeed(final CrawlURI curi) { if(getSeedsAsSurtPrefixes()) { surtPrefixes.add(prefixFrom(curi.getURI())); } diff --git a/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java index 1ba23eb1..5e5cc4f9 100644 --- a/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/AggressiveExtractorHTML.java @@ -1,34 +1,26 @@ /* - * AggressiveExtractorHTML + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * $Id$ + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Created on Jan 6, 2004 + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Copyright (C) 2004 Internet Archive. + * http://www.apache.org/licenses/LICENSE-2.0 * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package org.archive.modules.extractor; import java.util.logging.Logger; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Extended version of ExtractorHTML with more aggressive javascript link @@ -48,7 +40,7 @@ extends ExtractorHTML { public AggressiveExtractorHTML() { } - protected void processScript(ProcessorURI curi, CharSequence sequence, + protected void processScript(CrawlURI curi, CharSequence sequence, int endOfOpenTag) { super.processScript(curi, sequence, endOfOpenTag); // then, proccess entire javascript code as html code @@ -62,11 +54,11 @@ extends ExtractorHTML { */ public String report() { StringBuffer ret = new StringBuffer(256); - ret.append("Processor: "+AggressiveExtractorHTML.class.getName()+"\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on HTML documents " + "(including embedded CSS)\n"); ret.append(" ProcessorURRIs handled: " + numberOfCURIsHandled + "\n"); - ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); + ret.append(" Links extracted: " + numberOfLinksExtracted + "\n"); return ret.toString(); } } diff --git a/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java b/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java index 2de29a79..31720578 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/ContentExtractor.java @@ -1,29 +1,24 @@ -/* Copyright (C) 2006 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * ContentExtractor.java - * Created on October 5, 2006 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.extractor; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** @@ -37,7 +32,7 @@ public abstract class ContentExtractor extends Extractor { /** * Extracts links */ - final protected void extract(ProcessorURI uri) { + final protected void extract(CrawlURI uri) { boolean finished = innerExtract(uri); if (finished) { uri.linkExtractorFinished(); @@ -64,7 +59,7 @@ public abstract class ContentExtractor extends Extractor { * @return true if links should be extracted from the URI, * false otherwise */ - final protected boolean shouldProcess(ProcessorURI uri) { + final protected boolean shouldProcess(CrawlURI uri) { if (uri.hasBeenLinkExtracted()) { return false; } @@ -91,7 +86,7 @@ public abstract class ContentExtractor extends Extractor { * @return true if links should be extracted from that URI, false * otherwise */ - protected abstract boolean shouldExtract(ProcessorURI uri); + protected abstract boolean shouldExtract(CrawlURI uri); /** @@ -112,6 +107,6 @@ public abstract class ContentExtractor extends Extractor { * @return true if link extraction finished; false if downstream * extractors should attempt to extract links */ - protected abstract boolean innerExtract(ProcessorURI uri); + protected abstract boolean innerExtract(CrawlURI uri); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ContentExtractorTestBase.java b/modules/src/main/java/org/archive/modules/extractor/ContentExtractorTestBase.java index dff26cf2..9bfff9b4 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ContentExtractorTestBase.java +++ b/modules/src/main/java/org/archive/modules/extractor/ContentExtractorTestBase.java @@ -1,25 +1,20 @@ -/* Copyright (C) 2006 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * ContentExtractorTest.java - * Created on October 5, 2006 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.extractor; @@ -29,7 +24,7 @@ import java.io.File; import java.io.InputStream; import java.util.Collections; -import org.archive.modules.DefaultProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessorTestBase; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -76,14 +71,14 @@ public abstract class ContentExtractorTestBase extends ProcessorTestBase { /** - * Returns a DefaultProcessorURI for testing purposes. + * Returns a CrawlURI for testing purposes. * - * @return a DefaultProcessorURI + * @return a CrawlURI * @throws Exception just in case */ - protected DefaultProcessorURI defaultURI() throws Exception { + protected CrawlURI defaultURI() throws Exception { UURI uuri = UURIFactory.getInstance("http://www.archive.org/start/"); - return new DefaultProcessorURI(uuri, LinkContext.NAVLINK_MISC); + return new CrawlURI(uuri, null, null, LinkContext.NAVLINK_MISC); } @@ -93,7 +88,7 @@ public abstract class ContentExtractorTestBase extends ProcessorTestBase { * @throws Exception just in case */ public void testZeroContent() throws Exception { - DefaultProcessorURI uri = defaultURI(); + CrawlURI uri = defaultURI(); Recorder recorder = createRecorder(""); uri.setContentType("text/plain"); uri.setRecorder(recorder); @@ -110,7 +105,7 @@ public abstract class ContentExtractorTestBase extends ProcessorTestBase { * @throws Exception just in case */ public void testFinished() throws Exception { - DefaultProcessorURI uri = defaultURI(); + CrawlURI uri = defaultURI(); uri.linkExtractorFinished(); extractor.process(uri); assertEquals(0, uri.getOutLinks().size()); @@ -124,8 +119,7 @@ public abstract class ContentExtractorTestBase extends ProcessorTestBase { * * @param uri the URI to test */ - protected static void assertNoSideEffects(DefaultProcessorURI uri) { - assertEquals(0, uri.getUriErrors().size()); + protected static void assertNoSideEffects(CrawlURI uri) { assertEquals(0, uri.getNonFatalFailures().size()); assertEquals(Collections.EMPTY_LIST, uri.getAnnotations()); } diff --git a/modules/src/main/java/org/archive/modules/extractor/Extractor.java b/modules/src/main/java/org/archive/modules/extractor/Extractor.java index 2bb12b23..400eedc3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/Extractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/Extractor.java @@ -24,8 +24,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; +import org.archive.modules.CrawlURI; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -85,7 +85,7 @@ public abstract class Extractor extends Processor { * * @param uri the URI to extract links from */ - final protected void innerProcess(ProcessorURI uri) + final protected void innerProcess(CrawlURI uri) throws InterruptedException { try { extract(uri); @@ -100,7 +100,7 @@ public abstract class Extractor extends Processor { } - private void handleException(ProcessorURI uri, Throwable t) { + private void handleException(CrawlURI uri, Throwable t) { // both annotate (to highlight in crawl log) & add as local-error uri.getAnnotations().add("err=" + t.getClass().getName()); uri.getNonFatalFailures().add(t); @@ -119,7 +119,7 @@ public abstract class Extractor extends Processor { * * @param uri the uri to extract links from */ - protected abstract void extract(ProcessorURI uri); + protected abstract void extract(CrawlURI uri); public void logUriError(URIException e, UURI uuri, diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java index 5c750dab..2138a5fa 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorCSS.java @@ -25,7 +25,7 @@ import java.util.regex.Matcher; import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.util.ArchiveUtils; import org.archive.util.DevUtils; @@ -84,7 +84,7 @@ public class ExtractorCSS extends ContentExtractor { @Override - protected boolean shouldExtract(ProcessorURI curi) { + protected boolean shouldExtract(CrawlURI curi) { String mimeType = curi.getContentType(); if (mimeType == null) { return false; // FIXME: This check should be unnecessary @@ -99,7 +99,7 @@ public class ExtractorCSS extends ContentExtractor { /** * @param curi Crawl URI to process. */ - public boolean innerExtract(ProcessorURI curi) { + public boolean innerExtract(CrawlURI curi) { ReplayCharSequence cs = null; try { cs = curi.getRecorder().getReplayCharSequence(); @@ -116,7 +116,7 @@ public class ExtractorCSS extends ContentExtractor { } public static long processStyleCode(Extractor ext, - ProcessorURI curi, CharSequence cs) { + CrawlURI curi, CharSequence cs) { long foundLinks = 0; Matcher uris = null; String cssUri; @@ -148,10 +148,10 @@ public class ExtractorCSS extends ContentExtractor { public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.ExtractorCSS\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on Cascading Style Sheets (.css)\n"); ret.append(" ExtractorURIs handled: " + getURICount() + "\n"); - ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); + ret.append(" Links extracted: " + numberOfLinksExtracted + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java index edd3a425..32b3ee4b 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorDOC.java @@ -28,7 +28,7 @@ import org.apache.commons.io.IOUtils; import org.archive.io.ReplayInputStream; import org.archive.io.SeekReader; import org.archive.io.SeekReaderCharSequence; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.ms.Doc; @@ -58,7 +58,7 @@ public class ExtractorDOC extends ContentExtractor { @Override - protected boolean shouldExtract(ProcessorURI uri) { + protected boolean shouldExtract(CrawlURI uri) { String mimeType = uri.getContentType(); if (mimeType == null) { return false; @@ -71,9 +71,9 @@ public class ExtractorDOC extends ContentExtractor { * Processes a word document and extracts any hyperlinks from it. * This only extracts href style links, and does not examine the actual * text for valid URIs. - * @param curi ProcessorURI to process. + * @param curi CrawlURI to process. */ - protected boolean innerExtract(ProcessorURI curi){ + protected boolean innerExtract(CrawlURI curi){ int links = 0; ReplayInputStream documentStream = null; SeekReader docReader = null; @@ -108,7 +108,7 @@ public class ExtractorDOC extends ContentExtractor { } - private void addLink(ProcessorURI curi, String hyperlink) { + private void addLink(CrawlURI curi, String hyperlink) { try { UURI dest = UURIFactory.getInstance(curi.getUURI(), hyperlink); LinkContext lc = LinkContext.NAVLINK_MISC; @@ -125,10 +125,10 @@ public class ExtractorDOC extends ContentExtractor { */ public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.ExtractorDOC\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on MS Word documents (.doc)\n"); - ret.append(" ProcessorURIs handled: " + numberOfCURIsHandled + "\n"); - ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); + ret.append(" CrawlURIs handled: " + numberOfCURIsHandled + "\n"); + ret.append(" Links extracted: " + numberOfLinksExtracted + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java index 2ef2e9b9..b2e17384 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java @@ -29,7 +29,7 @@ import java.util.regex.Pattern; import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.net.RobotsHonoringPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -325,7 +325,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } - protected void processGeneralTag(ProcessorURI curi, CharSequence element, + protected void processGeneralTag(CrawlURI curi, CharSequence element, CharSequence cs) { Matcher attr = eachAttributeExtractor.matcher(cs); @@ -510,7 +510,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean * @param curi source CrawlURI * @param cs CharSequence of javascript code */ - protected void processScriptCode(ProcessorURI curi, CharSequence cs) { + protected void processScriptCode(CrawlURI curi, CharSequence cs) { if (getExtractJavascript()) { this.numberOfLinksExtracted += ExtractorJS.considerStrings(this, curi, cs, false); @@ -526,7 +526,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean * @param value * @param context */ - protected void processLink(ProcessorURI curi, final CharSequence value, + protected void processLink(CrawlURI curi, final CharSequence value, CharSequence context) { if (TextUtils.matches(JAVASCRIPT, value)) { processScriptCode(curi, value. subSequence(11, value.length())); @@ -542,7 +542,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } } - private void addLinkFromString(ProcessorURI curi, String uri, + private void addLinkFromString(CrawlURI curi, String uri, CharSequence context, Hop hop) { try { // We do a 'toString' on context because its a sequence from @@ -557,12 +557,12 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } } - protected final void processEmbed(ProcessorURI curi, CharSequence value, + protected final void processEmbed(CrawlURI curi, CharSequence value, CharSequence context) { processEmbed(curi, value, context, Hop.EMBED); } - protected void processEmbed(ProcessorURI curi, final CharSequence value, + protected void processEmbed(CrawlURI curi, final CharSequence value, CharSequence context, Hop hop) { if (logger.isLoggable(Level.FINEST)) { logger.finest("embed (" + hop.getHopChar() + "): " + value.toString() + @@ -576,7 +576,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } - protected boolean shouldExtract(ProcessorURI uri) { + protected boolean shouldExtract(CrawlURI uri) { if (getIgnoreUnexpectedHtml()) { try { // HTML was not expected (eg a GIF was expected) so ignore @@ -599,7 +599,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean } - public boolean innerExtract(ProcessorURI curi) { + public boolean innerExtract(CrawlURI curi) { this.numberOfCURIsHandled++; ReplayCharSequence cs = null; try { @@ -621,12 +621,12 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean /** * Run extractor. * This method is package visible to ease testing. - * @param curi ProcessorURI we're processing. + * @param curi CrawlURI we're processing. * @param cs Sequence from underlying ReplayCharSequence. This * is TRANSIENT data. Make a copy if you want the data to live outside * of this extractors' lifetime. */ - void extract(ProcessorURI curi, CharSequence cs) { + void extract(CrawlURI curi, CharSequence cs) { Matcher tags = relevantTagExtractor.matcher(cs); while(tags.find()) { if(Thread.interrupted()){ @@ -696,11 +696,11 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean * Test whether this HTML is so unexpected (eg in place of a GIF URI) * that it shouldn't be scanned for links. * - * @param curi ProcessorURI to examine. + * @param curi CrawlURI to examine. * @return True if HTML is acceptable/expected here * @throws URIException */ - protected boolean isHtmlExpectedHere(ProcessorURI curi) throws URIException { + protected boolean isHtmlExpectedHere(CrawlURI curi) throws URIException { String path = curi.getUURI().getPath(); if(path==null) { // no path extension, HTML is fine @@ -719,7 +719,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean return ! TextUtils.matches(NON_HTML_PATH_EXTENSION, ext); } - protected void processScript(ProcessorURI curi, CharSequence sequence, + protected void processScript(CrawlURI curi, CharSequence sequence, int endOfOpenTag) { // first, get attributes of script-open tag // as per any other tag @@ -734,13 +734,13 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean /** * Process metadata tags. - * @param curi ProcessorURI we're processing. + * @param curi CrawlURI we're processing. * @param cs Sequence from underlying ReplayCharSequence. This * is TRANSIENT data. Make a copy if you want the data to live outside * of this extractors' lifetime. * @return True robots exclusion metatag. */ - protected boolean processMeta(ProcessorURI curi, CharSequence cs) { + protected boolean processMeta(CrawlURI curi, CharSequence cs) { Matcher attr = eachAttributeExtractor.matcher(cs); String name = null; String httpEquiv = null; @@ -796,13 +796,13 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean /** * Process style text. - * @param curi ProcessorURI we're processing. + * @param curi CrawlURI we're processing. * @param sequence Sequence from underlying ReplayCharSequence. This * is TRANSIENT data. Make a copy if you want the data to live outside * of this extractors' lifetime. * @param endOfOpenTag */ - protected void processStyle(ProcessorURI curi, CharSequence sequence, + protected void processStyle(CrawlURI curi, CharSequence sequence, int endOfOpenTag) { // First, get attributes of script-open tag as per any other tag. processGeneralTag(curi, sequence.subSequence(0,6), @@ -822,11 +822,10 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean */ public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.ExtractorHTML\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on HTML documents\n"); - ret.append(" ProcessorURIs handled: " + this.numberOfCURIsHandled + "\n"); - ret.append(" Links extracted: " + this.numberOfLinksExtracted + - "\n\n"); + ret.append(" CrawlURIs handled: " + this.numberOfCURIsHandled + "\n"); + ret.append(" Links extracted: " + this.numberOfLinksExtracted + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java index b16abe43..f1e8f4e7 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTTP.java @@ -1,33 +1,29 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * SimpleHTTPExtractor.java - * Created on Jul 3, 2003 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + package org.archive.modules.extractor; import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.URIException; -import org.archive.modules.ProcessorURI; -import org.archive.modules.ProcessorURI.FetchType; +import org.archive.modules.CrawlURI; +import org.archive.modules.CrawlURI.FetchType; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -49,7 +45,7 @@ public class ExtractorHTTP extends Extractor { @Override - protected boolean shouldProcess(ProcessorURI uri) { + protected boolean shouldProcess(CrawlURI uri) { if (uri.getFetchStatus() <= 0) { return false; } @@ -59,13 +55,13 @@ public class ExtractorHTTP extends Extractor { @Override - protected void extract(ProcessorURI curi) { + protected void extract(CrawlURI curi) { HttpMethod method = curi.getHttpMethod(); addHeaderLink(curi, method.getResponseHeader("Location")); addHeaderLink(curi, method.getResponseHeader("Content-Location")); } - protected void addHeaderLink(ProcessorURI curi, Header loc) { + protected void addHeaderLink(CrawlURI curi, Header loc) { if (loc == null) { // If null, return without adding anything. return; @@ -85,11 +81,11 @@ public class ExtractorHTTP extends Extractor { public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.ExtractorHTTP\n"); + ret.append(super.report()); ret.append(" Function: " + "Extracts URIs from HTTP response headers\n"); ret.append(" CrawlURIs handled: " + this.getURICount()); - ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); + ret.append(" Links extracted: " + numberOfLinksExtracted + "\n"); return ret.toString(); } } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java index f8c15db0..918d749d 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorImpliedURI.java @@ -1,29 +1,21 @@ /* - * ExtractorURI + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * $Id$ + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Created on July 20, 2006 + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Copyright (C) 2006 Internet Archive. + * http://www.apache.org/licenses/LICENSE-2.0 * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package org.archive.modules.extractor; import java.util.ArrayList; @@ -35,7 +27,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.httpclient.URIException; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -121,7 +113,7 @@ public class ExtractorImpliedURI extends Extractor { @Override - protected boolean shouldProcess(ProcessorURI uri) { + protected boolean shouldProcess(CrawlURI uri) { return true; } @@ -131,7 +123,7 @@ public class ExtractorImpliedURI extends Extractor { * @param curi Crawl URI to process. */ @Override - public void extract(ProcessorURI curi) { + public void extract(CrawlURI curi) { List links = new ArrayList(curi.getOutLinks()); int max = links.size(); for (int i = 0; i < max; i++) { @@ -194,10 +186,10 @@ public class ExtractorImpliedURI extends Extractor { public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: "+ExtractorImpliedURI.class.getName()+"\n"); + ret.append(super.report()); ret.append(" Function: Extracts links inside other URIs\n"); ret.append(" CrawlURIs handled: " + getURICount() + "\n"); - ret.append(" Links extracted: " + linksExtracted.get() + "\n\n"); + ret.append(" Links extracted: " + linksExtracted.get() + "\n"); return ret.toString(); } } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index abd670b3..97f3cf10 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -16,7 +16,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.archive.modules.extractor; import java.io.IOException; @@ -27,7 +26,7 @@ import java.util.regex.Matcher; import org.apache.commons.codec.DecoderException; import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.LaxURLCodec; import org.archive.net.UURI; import org.archive.util.ArchiveUtils; @@ -96,7 +95,7 @@ public class ExtractorJS extends ContentExtractor { } - protected boolean shouldExtract(ProcessorURI uri) { + protected boolean shouldExtract(CrawlURI uri) { // special-cases, for when we know our current JS extractor does poorly. // TODO: remove this test when JS extractor is improved @@ -137,7 +136,7 @@ public class ExtractorJS extends ContentExtractor { @Override - protected boolean innerExtract(ProcessorURI curi) { + protected boolean innerExtract(CrawlURI curi) { this.numberOfCURIsHandled++; ReplayCharSequence cs = null; try { @@ -159,7 +158,7 @@ public class ExtractorJS extends ContentExtractor { } public static long considerStrings(Extractor ext, - ProcessorURI curi, CharSequence cs, boolean handlingJSFile) { + CrawlURI curi, CharSequence cs, boolean handlingJSFile) { long foundLinks = 0; Matcher strings = TextUtils.getMatcher(JAVASCRIPT_STRING_EXTRACTOR, cs); @@ -221,7 +220,7 @@ public class ExtractorJS extends ContentExtractor { * @return String changed/decoded to increase liklihood it is a * meaningful non-404 URI */ - public static String speculativeFixup(String string, ProcessorURI puri) { + public static String speculativeFixup(String string, CrawlURI puri) { String retVal = string; // unescape ampersands @@ -270,10 +269,10 @@ public class ExtractorJS extends ContentExtractor { */ public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.ExtractorJS\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on JavaScript code\n"); - ret.append(" ProcessorURIs handled: " + numberOfCURIsHandled + "\n"); - ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); + ret.append(" CrawlURIs handled: " + numberOfCURIsHandled + "\n"); + ret.append(" Links extracted: " + numberOfLinksExtracted + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java index 25e422a1..ff6671c3 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorPDF.java @@ -29,7 +29,7 @@ import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; import org.archive.io.SinkHandlerLogThread; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -66,7 +66,7 @@ public class ExtractorPDF extends ContentExtractor { } @Override - protected boolean shouldExtract(ProcessorURI uri) { + protected boolean shouldExtract(CrawlURI uri) { long max = getMaxSizeToParse(); if (uri.getRecorder().getRecordedInput().getSize() > max) { return false; @@ -77,7 +77,7 @@ public class ExtractorPDF extends ContentExtractor { } - protected boolean innerExtract(ProcessorURI curi){ + protected boolean innerExtract(CrawlURI curi){ File tempFile; int sn; @@ -145,10 +145,10 @@ public class ExtractorPDF extends ContentExtractor { */ public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.ExtractorPDF\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on PDF documents\n"); ret.append(" CrawlURIs handled: " + getURICount() + "\n"); - ret.append(" Links extracted: " + numberOfLinksExtracted + "\n\n"); + ret.append(" Links extracted: " + numberOfLinksExtracted + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java index d64c7682..cdf9cab4 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorSWF.java @@ -32,7 +32,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Logger; import java.util.regex.Matcher; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.util.TextUtils; import com.anotherbigidea.flash.interfaces.SWFActions; @@ -75,7 +75,7 @@ public class ExtractorSWF extends ContentExtractor { @Override - protected boolean shouldExtract(ProcessorURI uri) { + protected boolean shouldExtract(CrawlURI uri) { String contentType = uri.getContentType(); if (contentType == null) { return false; @@ -89,7 +89,7 @@ public class ExtractorSWF extends ContentExtractor { @Override - protected boolean innerExtract(ProcessorURI curi) { + protected boolean innerExtract(CrawlURI curi) { InputStream documentStream = null; // Get the SWF file's content stream. try { @@ -129,12 +129,12 @@ public class ExtractorSWF extends ContentExtractor { public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.ExtractorSWF\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on Shockwave Flash " + "documents (.swf)\n"); ret.append(" CrawlURIs handled: " + getURICount() + "\n"); - ret.append(" Links extracted: " + linksExtracted + "\n\n"); + ret.append(" Links extracted: " + linksExtracted + "\n"); return ret.toString(); } @@ -312,7 +312,7 @@ public class ExtractorSWF extends ContentExtractor { */ public class CrawlUriSWFAction extends SWFActionsImpl { - ProcessorURI curi; + CrawlURI curi; private long linkCount; private Extractor ext; @@ -321,7 +321,7 @@ public class ExtractorSWF extends ContentExtractor { * * @param curi */ - public CrawlUriSWFAction(ProcessorURI curi, Extractor ext) { + public CrawlUriSWFAction(CrawlURI curi, Extractor ext) { assert (curi != null) : "CrawlURI should not be null"; this.curi = curi; this.linkCount = 0; diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java index 23ee4375..01466757 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorURI.java @@ -1,29 +1,21 @@ /* - * ExtractorURI + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * $Id$ + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Created on July 20, 2006 + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Copyright (C) 2006 Internet Archive. + * http://www.apache.org/licenses/LICENSE-2.0 * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package org.archive.modules.extractor; import java.util.ArrayList; @@ -35,7 +27,7 @@ import java.util.regex.Matcher; import org.apache.commons.codec.DecoderException; import org.apache.commons.httpclient.URIException; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.LaxURLCodec; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -77,7 +69,7 @@ public class ExtractorURI extends Extractor { @Override - protected boolean shouldProcess(ProcessorURI uri) { + protected boolean shouldProcess(CrawlURI uri) { return true; } @@ -87,7 +79,7 @@ public class ExtractorURI extends Extractor { * @param curi Crawl URI to process. */ @Override - public void extract(ProcessorURI curi) { + public void extract(CrawlURI curi) { List links = new ArrayList(curi.getOutLinks()); int max = links.size(); for (int i = 0; i < max; i++) { @@ -102,7 +94,7 @@ public class ExtractorURI extends Extractor { * @param curi CrawlURI to add discoveries to * @param wref Link to examine for internal URIs */ - protected void extractLink(ProcessorURI curi, Link wref) { + protected void extractLink(CrawlURI curi, Link wref) { UURI source = null; try { source = UURIFactory.getInstance(wref.getDestination().toString()); @@ -181,10 +173,10 @@ public class ExtractorURI extends Extractor { public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: "+ExtractorURI.class.getName()+"\n"); + ret.append(super.report()); ret.append(" Function: Extracts links inside other URIs\n"); ret.append(" CrawlURIs handled: " + getURICount() + "\n"); - ret.append(" Links extracted: " + linksExtracted + "\n\n"); + ret.append(" Links extracted: " + linksExtracted + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java index 73cbee0a..b97d5059 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorUniversal.java @@ -1,23 +1,20 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Created on Jan 15, 2004 + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.extractor; @@ -29,7 +26,7 @@ import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -359,13 +356,13 @@ public class ExtractorUniversal extends ContentExtractor { @Override - protected boolean shouldExtract(ProcessorURI uri) { + protected boolean shouldExtract(CrawlURI uri) { return true; } @Override - protected boolean innerExtract(ProcessorURI curi) { + protected boolean innerExtract(CrawlURI curi) { InputStream instream = null; try { instream = curi.getRecorder().getRecordedInput(). @@ -538,12 +535,11 @@ public class ExtractorUniversal extends ContentExtractor { */ public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor." + - "ExtractorUniversal\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on unknown file" + " types.\n"); ret.append(" CrawlURIs handled: " + getURICount() + "\n"); - ret.append(" Links extracted: " + linksExtracted + "\n\n"); + ret.append(" Links extracted: " + linksExtracted + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java index c32e041a..03b24b44 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorXML.java @@ -1,29 +1,21 @@ /* - * ExtractorXML + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * $Id$ + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Created on Sep 27, 2005 + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Copyright (C) 2005 Internet Archive. + * http://www.apache.org/licenses/LICENSE-2.0 * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package org.archive.modules.extractor; import java.io.IOException; @@ -34,7 +26,7 @@ import java.util.regex.Pattern; import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; import org.archive.util.TextUtils; @@ -71,7 +63,7 @@ public class ExtractorXML extends ContentExtractor { @Override - protected boolean shouldExtract(ProcessorURI curi) { + protected boolean shouldExtract(CrawlURI curi) { String mimeType = curi.getContentType(); if (mimeType == null) { return false; @@ -90,7 +82,7 @@ public class ExtractorXML extends ContentExtractor { * @param curi Crawl URI to process. */ @Override - protected boolean innerExtract(ProcessorURI curi) { + protected boolean innerExtract(CrawlURI curi) { ReplayCharSequence cs = null; try { cs = curi.getRecorder().getReplayCharSequence(); @@ -106,7 +98,7 @@ public class ExtractorXML extends ContentExtractor { } public static long processXml(Extractor ext, - ProcessorURI curi, CharSequence cs) { + CrawlURI curi, CharSequence cs) { long foundLinks = 0; Matcher uris = null; String xmlUri; @@ -134,10 +126,10 @@ public class ExtractorXML extends ContentExtractor { public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.ExtractorXML\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on XML/RSS\n"); ret.append(" CrawlURIs handled: " + getURICount() + "\n"); - ret.append(" Links extracted: " + linksExtracted + "\n\n"); + ret.append(" Links extracted: " + linksExtracted + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java index 2cf6fded..2d7299f2 100644 --- a/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java +++ b/modules/src/main/java/org/archive/modules/extractor/HTTPContentDigest.java @@ -26,7 +26,7 @@ import java.util.regex.Pattern; import org.archive.io.ReplayCharSequence; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; /** @@ -100,7 +100,7 @@ public class HTTPContentDigest extends Processor { } - protected boolean shouldProcess(ProcessorURI uri) { + protected boolean shouldProcess(CrawlURI uri) { if (!uri.getContentType().startsWith("text")) { return false; } @@ -113,7 +113,7 @@ public class HTTPContentDigest extends Processor { return true; } - protected void innerProcess(ProcessorURI curi) throws InterruptedException { + protected void innerProcess(CrawlURI curi) throws InterruptedException { // Ok, if we got this far we need to calculate the content digest. // Get the regexpr Pattern regexpr = getStripRegex(); diff --git a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java index 68ba1bba..dafed432 100644 --- a/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java +++ b/modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java @@ -35,7 +35,7 @@ import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.net.RobotsHonoringPolicy; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -113,7 +113,7 @@ public class JerichoExtractorHTML extends ExtractorHTML { } - protected void processGeneralTag(ProcessorURI curi, Element element, + protected void processGeneralTag(CrawlURI curi, Element element, Attributes attributes) { Attribute attr; String attrValue; @@ -282,7 +282,7 @@ public class JerichoExtractorHTML extends ExtractorHTML { } } - protected boolean processMeta(ProcessorURI curi, Element element) { + protected boolean processMeta(CrawlURI curi, Element element) { String name = element.getAttributeValue("name"); String httpEquiv = element.getAttributeValue("http-equiv"); String content = element.getAttributeValue("content"); @@ -316,7 +316,7 @@ public class JerichoExtractorHTML extends ExtractorHTML { return false; } - protected void processScript(ProcessorURI curi, Element element) { + protected void processScript(CrawlURI curi, Element element) { // first, get attributes of script-open tag // as per any other tag processGeneralTag(curi, element, element.getAttributes()); @@ -327,7 +327,7 @@ public class JerichoExtractorHTML extends ExtractorHTML { } - protected void processStyle(ProcessorURI curi, Element element) { + protected void processStyle(CrawlURI curi, Element element) { // First, get attributes of script-open tag as per any other tag. processGeneralTag(curi, element, element.getAttributes()); @@ -336,7 +336,7 @@ public class JerichoExtractorHTML extends ExtractorHTML { this, curi, element.getContent()); } - protected void processForm(ProcessorURI curi, Element element) { + protected void processForm(CrawlURI curi, Element element) { String action = element.getAttributeValue("action"); String name = element.getAttributeValue("name"); String queryURL = ""; @@ -411,11 +411,11 @@ public class JerichoExtractorHTML extends ExtractorHTML { * Run extractor. This method is package visible to ease testing. * * @param curi - * ProcessorURI we're processing. + * CrawlURI we're processing. * @param cs * Sequence from underlying ReplayCharSequence. */ - void extract(ProcessorURI curi, CharSequence cs) { + void extract(CrawlURI curi, CharSequence cs) { Source source = new Source(cs); List elements = source.findAllElements(StartTagType.NORMAL); for (Iterator elementIter = elements.iterator(); @@ -447,11 +447,11 @@ public class JerichoExtractorHTML extends ExtractorHTML { */ public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.JerichoExtractorHTML\n"); + ret.append(super.report()); ret.append(" Function: Link extraction on HTML documents\n"); - ret.append(" ProcessorURIs handled: " + this.numberOfCURIsHandled + "\n"); + ret.append(" CrawlURIs handled: " + this.numberOfCURIsHandled + "\n"); ret.append(" Forms processed: " + this.numberOfFormsProcessed + "\n"); - ret.append(" Links extracted: " + this.numberOfLinksExtracted + "\n\n"); + ret.append(" Links extracted: " + this.numberOfLinksExtracted + "\n"); return ret.toString(); } } diff --git a/modules/src/main/java/org/archive/modules/extractor/Link.java b/modules/src/main/java/org/archive/modules/extractor/Link.java index 3affdfe0..ab240be8 100644 --- a/modules/src/main/java/org/archive/modules/extractor/Link.java +++ b/modules/src/main/java/org/archive/modules/extractor/Link.java @@ -1,33 +1,27 @@ -/* Link -* -* $Id$ -* -* Created on Mar 7, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.extractor; import java.io.Serializable; import org.apache.commons.httpclient.URIException; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -42,7 +36,7 @@ import org.archive.net.UURIFactory; * * @author gojomo */ -public class Link implements Serializable { +public class Link implements Serializable, Comparable { private static final long serialVersionUID = 2L; @@ -130,28 +124,28 @@ public class Link implements Serializable { ^ context.hashCode() ^ hop.hashCode(); } - public static void addRelativeToBase(ProcessorURI uri, int max, + public static void addRelativeToBase(CrawlURI uri, int max, String newUri, LinkContext context, Hop hop) throws URIException { UURI dest = UURIFactory.getInstance(uri.getBaseURI(), newUri); add2(uri, max, dest, context, hop); } - public static void addRelativeToVia(ProcessorURI uri, int max, + public static void addRelativeToVia(CrawlURI uri, int max, String newUri, LinkContext context, Hop hop) throws URIException { UURI dest = UURIFactory.getInstance(uri.getVia(), newUri); add2(uri, max, dest, context, hop); } - public static void add(ProcessorURI uri, int max, String newUri, + public static void add(CrawlURI uri, int max, String newUri, LinkContext context, Hop hop) throws URIException { UURI dest = UURIFactory.getInstance(newUri); add2(uri, max, dest, context, hop); } - private static void add2(ProcessorURI uri, int max, UURI dest, + private static void add2(CrawlURI uri, int max, UURI dest, LinkContext context, Hop hop) throws URIException { if (uri.getOutLinks().size() < max) { UURI src = uri.getUURI(); @@ -163,4 +157,18 @@ public class Link implements Serializable { } } + public int compareTo(Link o) { + int cmp = source.toString().compareTo(o.source.toString()); + if (cmp==0) { + cmp = destination.toString().compareTo(o.destination.toString()); + } + if (cmp==0) { + cmp = context.toString().compareTo(o.context.toString()); + } + if (cmp==0) { + cmp = hop.toString().compareTo(o.hop.toString()); + } + return cmp; + } + } diff --git a/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java b/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java index 38e83114..abbc9769 100644 --- a/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java +++ b/modules/src/main/java/org/archive/modules/extractor/StringExtractorTestBase.java @@ -1,23 +1,37 @@ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.extractor; - -import java.util.ArrayList; import java.util.Collection; -import java.util.List; - -import org.archive.modules.DefaultProcessorURI; - +import java.util.HashSet; +import org.archive.modules.CrawlURI; public abstract class StringExtractorTestBase extends ContentExtractorTestBase { public static class TestData { - public DefaultProcessorURI uri; + public CrawlURI uri; public Link expectedResult; - public TestData(DefaultProcessorURI uri, Link expectedResult) { + public TestData(CrawlURI uri, Link expectedResult) { this.uri = uri; this.expectedResult = expectedResult; } @@ -67,7 +81,7 @@ public abstract class StringExtractorTestBase extends ContentExtractorTestBase { for (TestData testData: testDataCol) { Extractor extractor = makeExtractor(); extractor.process(testData.uri); - List expected = new ArrayList(); + HashSet expected = new HashSet(); expected.add(testData.expectedResult); assertEquals(expected, testData.uri.getOutLinks()); assertNoSideEffects(testData.uri); diff --git a/modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java b/modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java index 42f13f11..7ae56431 100644 --- a/modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java +++ b/modules/src/main/java/org/archive/modules/extractor/TrapSuppressExtractor.java @@ -1,27 +1,24 @@ -/* Copyright (C) 2007 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Created on Mar 5, 2007 + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.extractor; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Pseudo-extractor that suppresses link-extraction of likely trap pages, @@ -46,11 +43,11 @@ public class TrapSuppressExtractor extends ContentExtractor { public TrapSuppressExtractor() { } - protected boolean shouldExtract(ProcessorURI uri) { + protected boolean shouldExtract(CrawlURI uri) { return true; } - protected boolean innerExtract(ProcessorURI curi){ + protected boolean innerExtract(CrawlURI curi){ numberOfCURIsHandled++; String currentDigest = curi.getContentDigestSchemeString(); @@ -81,10 +78,10 @@ public class TrapSuppressExtractor extends ContentExtractor { */ public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.extractor.TrapSuppressExtractor\n"); + ret.append(super.report()); ret.append(" Function: Suppress extraction on likely traps\n"); ret.append(" CrawlURIs handled: " + numberOfCURIsHandled + "\n"); - ret.append(" CrawlURIs suppressed: " + numberOfCURIsSuppressed + "\n\n"); + ret.append(" CrawlURIs suppressed: " + numberOfCURIsSuppressed + "\n"); return ret.toString(); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java index 265ac704..8acb1fcb 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchDNS.java @@ -1,25 +1,20 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * FetchDNS - * Created on Jun 5, 2003 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.fetcher; @@ -41,7 +36,7 @@ import java.util.regex.Matcher; import org.apache.commons.httpclient.URIException; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; import org.archive.util.ArchiveUtils; @@ -133,12 +128,12 @@ public class FetchDNS extends Processor { public FetchDNS() { } - protected boolean shouldProcess(ProcessorURI curi) { + protected boolean shouldProcess(CrawlURI curi) { return curi.getUURI().getScheme().equals("dns"); } - protected void innerProcess(ProcessorURI curi) { + protected void innerProcess(CrawlURI curi) { Record[] rrecordSet = null; // Retrieved dns records String dnsName = null; try { @@ -207,7 +202,7 @@ public class FetchDNS extends Processor { curi.setFetchCompletedTime(System.currentTimeMillis()); } - protected void storeDNSRecord(final ProcessorURI curi, final String dnsName, + protected void storeDNSRecord(final CrawlURI curi, final String dnsName, final CrawlHost targetHost, final Record[] rrecordSet) { // Get TTL and IP info from the first A record (there may be // multiple, e.g. www.washington.edu) then update the CrawlServer @@ -228,7 +223,7 @@ public class FetchDNS extends Processor { } } - protected boolean isQuadAddress(final ProcessorURI curi, final String dnsName, + protected boolean isQuadAddress(final CrawlURI curi, final String dnsName, final CrawlHost targetHost) { boolean result = false; Matcher matcher = InetAddressUtil.IPV4_QUADS.matcher(dnsName); @@ -238,10 +233,10 @@ public class FetchDNS extends Processor { } result = true; - // Ideally this branch would never be reached: no ProcessorURI + // Ideally this branch would never be reached: no CrawlURI // would be created for numerical IPs if (logger.isLoggable(Level.WARNING)) { - logger.warning("Unnecessary DNS ProcessorURI created: " + curi); + logger.warning("Unnecessary DNS CrawlURI created: " + curi); } try { targetHost.setIP(InetAddress.getByAddress(dnsName, new byte[] { @@ -258,7 +253,7 @@ public class FetchDNS extends Processor { return result; } - protected void recordDNS(final ProcessorURI curi, final Record[] rrecordSet) + protected void recordDNS(final CrawlURI curi, final Record[] rrecordSet) throws IOException { final byte[] dnsRecord = getDNSRecord(curi.getFetchBeginTime(), rrecordSet); @@ -321,7 +316,7 @@ public class FetchDNS extends Processor { return baos.toByteArray(); } - protected void setUnresolvable(ProcessorURI curi, CrawlHost host) { + protected void setUnresolvable(CrawlURI curi, CrawlHost host) { host.setIP(null, 0); curi.setFetchStatus(S_DOMAIN_UNRESOLVABLE); } diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java index ee37d5cb..4564add0 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchFTP.java @@ -1,26 +1,20 @@ -/* FetchFTP.java +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * $Id$ + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Created on Jun 5, 2003 + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Copyright (C) 2003 Internet Archive. + * http://www.apache.org/licenses/LICENSE-2.0 * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.fetcher; @@ -38,8 +32,8 @@ import org.apache.commons.httpclient.URIException; import org.apache.commons.net.ftp.FTPCommand; import org.archive.io.RecordingInputStream; import org.archive.io.ReplayCharSequence; +import org.archive.modules.CrawlURI; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import org.archive.modules.extractor.Hop; import org.archive.modules.extractor.Link; import org.archive.modules.extractor.LinkContext; @@ -189,7 +183,7 @@ public class FetchFTP extends Processor { @Override - protected boolean shouldProcess(ProcessorURI curi) { + protected boolean shouldProcess(CrawlURI curi) { if (!curi.getUURI().getScheme().equals("ftp")) { return false; } @@ -227,7 +221,7 @@ public class FetchFTP extends Processor { * processing */ @Override - protected void innerProcess(ProcessorURI curi) throws InterruptedException { + protected void innerProcess(CrawlURI curi) throws InterruptedException { curi.setFetchBeginTime(System.currentTimeMillis()); Recorder recorder = curi.getRecorder(); ClientFTP client = new ClientFTP(); @@ -257,7 +251,7 @@ public class FetchFTP extends Processor { * @throws IOException if a network or protocol error occurs * @throws InterruptedException if the thread is interrupted */ - private void fetch(ProcessorURI curi, ClientFTP client, Recorder recorder) + private void fetch(CrawlURI curi, ClientFTP client, Recorder recorder) throws IOException, InterruptedException { // Connect to the FTP server. UURI uuri = curi.getUURI(); @@ -322,7 +316,7 @@ public class FetchFTP extends Processor { * @throws IOException if a network or file error occurs * @throws InterruptedException if the thread is interrupted */ - private void saveToRecorder(ProcessorURI curi, + private void saveToRecorder(CrawlURI curi, Socket socket, Recorder recorder) throws IOException, InterruptedException { recorder.inputWrap(socket.getInputStream()); @@ -347,7 +341,7 @@ public class FetchFTP extends Processor { * @param curi The curi to save extracted links to * @param recorder The recorder containing the directory listing */ - private void extract(ProcessorURI curi, Recorder recorder) { + private void extract(CrawlURI curi, Recorder recorder) { if (!getExtractFromDirs()) { return; } @@ -373,7 +367,7 @@ public class FetchFTP extends Processor { * @param dir The directory listing to extract links from * @throws URIException if an extracted link is invalid */ - private void extract(ProcessorURI curi, ReplayCharSequence dir) { + private void extract(CrawlURI curi, ReplayCharSequence dir) { logger.log(Level.FINEST, "Extracting URIs from FTP directory."); Matcher matcher = DIR.matcher(dir); while (matcher.find()) { @@ -391,7 +385,7 @@ public class FetchFTP extends Processor { * @param curi the curi to store the discovered link in * @param file the filename of the discovered link */ - private void addExtracted(ProcessorURI curi, String file) { + private void addExtracted(CrawlURI curi, String file) { try { file = URLEncoder.encode(file, "UTF-8"); } catch (UnsupportedEncodingException e) { @@ -428,7 +422,7 @@ public class FetchFTP extends Processor { * * @param curi the curi whose parent to add */ - private void addParent(ProcessorURI curi) { + private void addParent(CrawlURI curi) { if (!getExtractParent()) { return; } @@ -469,7 +463,7 @@ public class FetchFTP extends Processor { * @param curi the curi whose username and password to return * @return an array containing the username and password */ - private String[] getAuth(ProcessorURI curi) { + private String[] getAuth(CrawlURI curi) { String[] result = new String[2]; UURI uuri = curi.getUURI(); String userinfo; diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java index 22513819..723d5157 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchHTTP.java @@ -19,7 +19,7 @@ package org.archive.modules.fetcher; -import static org.archive.modules.ProcessorURI.FetchType.HTTP_POST; +import static org.archive.modules.CrawlURI.FetchType.HTTP_POST; import static org.archive.modules.fetcher.FetchErrors.HEADER_TRUNC; import static org.archive.modules.fetcher.FetchErrors.LENGTH_TRUNC; import static org.archive.modules.fetcher.FetchErrors.TIMER_TRUNC; @@ -89,9 +89,9 @@ import org.archive.httpclient.ConfigurableX509TrustManager.TrustLevel; import org.archive.io.RecorderLengthExceededException; import org.archive.io.RecorderTimeoutException; import org.archive.io.RecorderTooMuchHeaderException; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import org.archive.modules.credential.Credential; import org.archive.modules.credential.CredentialAvatar; import org.archive.modules.credential.CredentialStore; @@ -503,7 +503,7 @@ public class FetchHTTP extends Processor implements Lifecycle { public FetchHTTP() { } - protected void innerProcess(final ProcessorURI curi) + protected void innerProcess(final CrawlURI curi) throws InterruptedException { // Note begin time curi.setFetchBeginTime(System.currentTimeMillis()); @@ -631,8 +631,8 @@ public class FetchHTTP extends Processor implements Lifecycle { } if (isSuccess(curi) && addedCredentials) { - // Promote the credentials from the ProcessorURI to the CrawlServer - // so they are available for all subsequent ProcessorURIs on this + // Promote the credentials from the CrawlURI to the CrawlServer + // so they are available for all subsequent CrawlURIs on this // server. promoteCredentials(curi); if (logger.isLoggable(Level.FINE)) { @@ -667,7 +667,7 @@ public class FetchHTTP extends Processor implements Lifecycle { * @param rec HttpRecorder */ @SuppressWarnings("unchecked") - protected void setSizes(ProcessorURI curi, Recorder rec) { + protected void setSizes(CrawlURI curi, Recorder rec) { // set reporting size curi.setContentSize(rec.getRecordedInput().getSize()); // special handling for 304-not modified @@ -687,14 +687,14 @@ public class FetchHTTP extends Processor implements Lifecycle { } } - protected void doAbort(ProcessorURI curi, HttpMethod method, + protected void doAbort(CrawlURI curi, HttpMethod method, String annotation) { curi.getAnnotations().add(annotation); curi.getRecorder().close(); method.abort(); } - protected boolean checkMidfetchAbort(ProcessorURI curi, + protected boolean checkMidfetchAbort(CrawlURI curi, HttpRecorderMethod method, HttpConnection conn) { if (curi.isPrerequisite()) { return false; @@ -712,11 +712,11 @@ public class FetchHTTP extends Processor implements Lifecycle { * content type. * * @param curi - * ProcessorURI to populate. + * CrawlURI to populate. * @param method * Method to get response status and headers from. */ - protected void addResponseContent(HttpMethod method, ProcessorURI curi) { + protected void addResponseContent(HttpMethod method, CrawlURI curi) { curi.setFetchStatus(method.getStatusCode()); Header ct = method.getResponseHeader("content-type"); curi.setContentType((ct == null) ? null : ct.getValue()); @@ -744,7 +744,7 @@ public class FetchHTTP extends Processor implements Lifecycle { * @param method * Method used for the request. */ - private void setCharacterEncoding(ProcessorURI uri, final Recorder rec, + private void setCharacterEncoding(CrawlURI uri, final Recorder rec, final HttpMethod method) { String encoding = null; encoding = ((HttpMethodBase) method).getResponseCharSet(); @@ -764,14 +764,14 @@ public class FetchHTTP extends Processor implements Lifecycle { * Cleanup after a failed method execute. * * @param curi - * ProcessorURI we failed on. + * CrawlURI we failed on. * @param method * Method we failed on. * @param exception * Exception we failed with. */ private void failedExecuteCleanup(final HttpMethod method, - final ProcessorURI curi, final Exception exception) { + final CrawlURI curi, final Exception exception) { cleanup(curi, exception, "executeMethod", S_CONNECT_FAILED); method.releaseConnection(); } @@ -780,7 +780,7 @@ public class FetchHTTP extends Processor implements Lifecycle { * Cleanup after a failed method execute. * * @param curi - * ProcessorURI we failed on. + * CrawlURI we failed on. * @param exception * Exception we failed with. * @param message @@ -788,7 +788,7 @@ public class FetchHTTP extends Processor implements Lifecycle { * @param status * Status to set on the fetch. */ - private void cleanup(final ProcessorURI curi, final Exception exception, + private void cleanup(final CrawlURI curi, final Exception exception, final String message, final int status) { // message ignored! curi.getNonFatalFailures().add(exception); @@ -797,7 +797,7 @@ public class FetchHTTP extends Processor implements Lifecycle { } @Override - public ProcessResult process(ProcessorURI uri) throws InterruptedException { + public ProcessResult process(CrawlURI uri) throws InterruptedException { if (uri.getFetchStatus() < 0) { // already marked as errored, this pass through // skip to end @@ -808,15 +808,15 @@ public class FetchHTTP extends Processor implements Lifecycle { } /** - * Can this processor fetch the given ProcessorURI. May set a fetch status - * if this processor would usually handle the ProcessorURI, but cannot in + * Can this processor fetch the given CrawlURI. May set a fetch status + * if this processor would usually handle the CrawlURI, but cannot in * this instance. * * @param curi * @return True if processor can fetch. */ @Override - protected boolean shouldProcess(ProcessorURI curi) { + protected boolean shouldProcess(CrawlURI curi) { String scheme = curi.getUURI().getScheme(); if (!(scheme.equals("http") || scheme.equals("https"))) { // handles only plain http and https @@ -836,11 +836,11 @@ public class FetchHTTP extends Processor implements Lifecycle { * Configure the HttpMethod setting options and headers. * * @param curi - * ProcessorURI from which we pull configuration. + * CrawlURI from which we pull configuration. * @param method * The Method to configure. */ - protected HostConfiguration configureMethod(ProcessorURI curi, + protected HostConfiguration configureMethod(CrawlURI curi, HttpMethod method) { // Don't auto-follow redirects method.setFollowRedirects(false); @@ -925,7 +925,7 @@ public class FetchHTTP extends Processor implements Lifecycle { * @param targetHeader header to set if possible */ @SuppressWarnings("unchecked") - protected void setConditionalGetHeader(ProcessorURI curi, HttpMethod method, + protected void setConditionalGetHeader(CrawlURI curi, HttpMethod method, boolean conditional, String sourceHeader, String targetHeader) { if (conditional) { try { @@ -946,10 +946,10 @@ public class FetchHTTP extends Processor implements Lifecycle { } /** - * Setup proxy, based on attributes in ProcessorURI and settings, + * Setup proxy, based on attributes in CrawlURI and settings, * in given HostConfiguration */ - private void configureProxy(ProcessorURI curi, HostConfiguration config) { + private void configureProxy(CrawlURI curi, HostConfiguration config) { String proxy = (String) getAttributeEither(curi, "httpProxyHost"); int port = (Integer) getAttributeEither(curi, "httpProxyPort"); configureProxy(proxy, port, config); @@ -963,10 +963,10 @@ public class FetchHTTP extends Processor implements Lifecycle { } /** - * Setup local bind address, based on attributes in ProcessorURI and + * Setup local bind address, based on attributes in CrawlURI and * settings, in given HostConfiguration */ - private void configureBindAddress(ProcessorURI curi, HostConfiguration config) { + private void configureBindAddress(CrawlURI curi, HostConfiguration config) { String addressString = (String) getAttributeEither(curi, HTTP_BIND_ADDRESS); configureBindAddress(addressString,config); } @@ -986,16 +986,16 @@ public class FetchHTTP extends Processor implements Lifecycle { } /** - * Get a value either from inside the ProcessorURI instance, or from + * Get a value either from inside the CrawlURI instance, or from * settings (module attributes). * * @param curi - * ProcessorURI to consult + * CrawlURI to consult * @param key * key to lookup - * @return value from either ProcessorURI (preferred) or settings + * @return value from either CrawlURI (preferred) or settings */ - protected Object getAttributeEither(ProcessorURI curi, String key) { + protected Object getAttributeEither(CrawlURI curi, String key) { Object r = curi.getData().get(key); if (r != null) { @@ -1014,18 +1014,18 @@ public class FetchHTTP extends Processor implements Lifecycle { * Returns true if found credentials to be tried. * * @param curi - * Current ProcessorURI. + * Current CrawlURI. * @param method * The method to add to. * @return True if prepopulated method with credentials AND * the credentials came from the curi, not from the * CrawlServer. The former is special in that if the * curi credentials - * succeed, then the caller needs to promote them from the ProcessorURI to the - * CrawlServer so they are available for all subsequent ProcessorURIs on this + * succeed, then the caller needs to promote them from the CrawlURI to the + * CrawlServer so they are available for all subsequent CrawlURIs on this * server. */ - private boolean populateCredentials(ProcessorURI curi, HttpMethod method) { + private boolean populateCredentials(CrawlURI curi, HttpMethod method) { // First look at the server avatars. Add any that are to be volunteered // on every request (e.g. RFC2617 credentials). Every time creds will // return true when we call 'isEveryTime(). @@ -1064,9 +1064,9 @@ public class FetchHTTP extends Processor implements Lifecycle { * Promote successful credential to the server. * * @param curi - * ProcessorURI whose credentials we are to promote. + * CrawlURI whose credentials we are to promote. */ - private void promoteCredentials(final ProcessorURI curi) { + private void promoteCredentials(final CrawlURI curi) { Set avatars = curi.getCredentialAvatars(); for (Iterator i = avatars.iterator(); i.hasNext();) { CredentialAvatar ca = i.next(); @@ -1089,7 +1089,7 @@ public class FetchHTTP extends Processor implements Lifecycle { /** * Server is looking for basic/digest auth credentials (RFC2617). If we have - * any, put them into the ProcessorURI and have it come around again. + * any, put them into the CrawlURI and have it come around again. * Presence of the credential serves as flag to frontier to requeue * promptly. If we already tried this domain and still got a 401, then our * credentials are bad. Remove them and let this curi die. @@ -1097,9 +1097,9 @@ public class FetchHTTP extends Processor implements Lifecycle { * @param method * Method that got a 401. * @param curi - * ProcessorURI that got a 401. + * CrawlURI that got a 401. */ - protected void handle401(final HttpMethod method, final ProcessorURI curi) { + protected void handle401(final HttpMethod method, final CrawlURI curi) { AuthScheme authscheme = getAuthScheme(method, curi); if (authscheme == null) { return; @@ -1168,12 +1168,12 @@ public class FetchHTTP extends Processor implements Lifecycle { * @param method * Method that got a 401. * @param curi - * ProcessorURI that got a 401. + * CrawlURI that got a 401. * @return Returns first wholesome authscheme found else null. */ @SuppressWarnings("unchecked") protected AuthScheme getAuthScheme(final HttpMethod method, - final ProcessorURI curi) { + final CrawlURI curi) { Header[] headers = method.getResponseHeaders("WWW-Authenticate"); if (headers == null || headers.length <= 0) { logger.info("We got a 401 but no WWW-Authenticate challenge: " @@ -1238,12 +1238,12 @@ public class FetchHTTP extends Processor implements Lifecycle { /** * @param curi - * ProcessorURI that got a 401. + * CrawlURI that got a 401. * @param type * Class of credential to get from curi. * @return Set of credentials attached to this curi. */ - private Set getCredentials(ProcessorURI curi, Class type) { + private Set getCredentials(CrawlURI curi, Class type) { Set result = null; if (curi.hasCredentialAvatars()) { @@ -1387,16 +1387,16 @@ public class FetchHTTP extends Processor implements Lifecycle { */ public String report() { StringBuffer ret = new StringBuffer(); - ret.append("Processor: org.archive.crawler.fetcher.FetchHTTP\n"); + ret.append(super.report()); ret.append(" Function: Fetch HTTP URIs\n"); - ret.append(" ProcessorURIs handled: " + this.getURICount() + "\n"); - ret.append(" Recovery retries: " + this.recoveryRetries + "\n\n"); + ret.append(" CrawlURIs handled: " + this.getURICount() + "\n"); + ret.append(" Recovery retries: " + this.recoveryRetries + "\n"); return ret.toString(); } - private void setAcceptHeaders(ProcessorURI curi, HttpMethod get) { + private void setAcceptHeaders(CrawlURI curi, HttpMethod get) { List acceptHeaders = getAcceptHeaders(); if (acceptHeaders.isEmpty()) { return; @@ -1490,7 +1490,7 @@ public class FetchHTTP extends Processor implements Lifecycle { return this.http; } - private static String getServerKey(ProcessorURI uri) { + private static String getServerKey(CrawlURI uri) { try { return CrawlServer.getServerKey(uri.getUURI()); } catch (URIException e) { diff --git a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java index aeadb810..66e85b8d 100644 --- a/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java +++ b/modules/src/main/java/org/archive/modules/fetcher/FetchStats.java @@ -1,33 +1,27 @@ -/* CrawlSubstats -* -* $Id$ -* -* Created on Nov 4, 2005 -* -* Copyright (C) 2005 Internet Archive. -* -* This file is part of the Heritrix web crawler (crawler.archive.org). -* -* Heritrix is free software; you can redistribute it and/or modify -* it under the terms of the GNU Lesser Public License as published by -* the Free Software Foundation; either version 2.1 of the License, or -* any later version. -* -* Heritrix is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Lesser Public License for more details. -* -* You should have received a copy of the GNU Lesser Public License -* along with Heritrix; if not, write to the Free Software -* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -*/ +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). + * + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.archive.modules.fetcher; import java.io.PrintWriter; import java.io.Serializable; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; import org.archive.util.MultiReporter; @@ -48,7 +42,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter public FetchStats getSubstats(); } public interface CollectsFetchStats { - public void tally(ProcessorURI curi, Stage stage); + public void tally(CrawlURI curi, Stage stage); } long totalScheduled; // anything initially scheduled @@ -64,7 +58,7 @@ public class FetchStats implements Serializable, FetchStatusCodes, MultiReporter long fetchNonResponses; // processing attempts resulting in no response // (both failures and temp deferrals) - public synchronized void tally(ProcessorURI curi, Stage stage) { + public synchronized void tally(CrawlURI curi, Stage stage) { switch(stage) { case SCHEDULED: totalScheduled++; diff --git a/modules/src/main/java/org/archive/modules/net/CrawlServer.java b/modules/src/main/java/org/archive/modules/net/CrawlServer.java index 897610d7..4aa17b81 100644 --- a/modules/src/main/java/org/archive/modules/net/CrawlServer.java +++ b/modules/src/main/java/org/archive/modules/net/CrawlServer.java @@ -1,25 +1,20 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * CrawlServer.java - * Created on Apr 17, 2003 - * - * $Header$ + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.net; @@ -35,10 +30,10 @@ import java.util.zip.Checksum; import org.apache.commons.httpclient.URIException; import org.apache.commons.io.IOUtils; -import static org.archive.modules.ProcessorURI.FetchType.*; +import static org.archive.modules.CrawlURI.FetchType.*; import org.archive.io.ReplayInputStream; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.credential.CredentialAvatar; import org.archive.modules.fetcher.FetchStats; import org.archive.net.UURI; @@ -146,9 +141,9 @@ public class CrawlServer implements Serializable, FetchStats.HasFetchStats { * @throws IOException */ public synchronized void updateRobots(RobotsHonoringPolicy honoringPolicy, - ProcessorURI curi) { + CrawlURI curi) { - ProcessorURI.FetchType ft = curi.getFetchType(); + CrawlURI.FetchType ft = curi.getFetchType(); boolean gotSomething = curi.getFetchStatus() > 0 && (ft == HTTP_GET || ft == HTTP_POST); if (!gotSomething && curi.getFetchAttempts() < MIN_ROBOTS_RETRIES) { diff --git a/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java b/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java index 189de727..7e49dfe3 100644 --- a/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java +++ b/modules/src/main/java/org/archive/modules/net/RobotsExclusionPolicy.java @@ -31,7 +31,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.httpclient.URIException; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * RobotsExclusionPolicy represents the actual policy adopted with @@ -140,7 +140,7 @@ public class RobotsExclusionPolicy implements Serializable { this.type = type; } - public boolean disallows(ProcessorURI curi, String userAgent) { + public boolean disallows(CrawlURI curi, String userAgent) { if (this == ALLOWALL) return false; if (this == DENYALL) diff --git a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java index d8ceb63b..4a776817 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/FetchHistoryProcessor.java @@ -25,7 +25,6 @@ import org.apache.commons.httpclient.Header; import org.apache.commons.httpclient.HttpMethod; import org.archive.modules.CrawlURI; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import static org.archive.modules.recrawl.RecrawlAttributeConstants.*; import static org.archive.modules.ModuleAttributeConstants.A_FETCH_BEGAN_TIME; @@ -60,7 +59,7 @@ public class FetchHistoryProcessor extends Processor { @SuppressWarnings("unchecked") @Override - protected void innerProcess(ProcessorURI puri) throws InterruptedException { + protected void innerProcess(CrawlURI puri) throws InterruptedException { CrawlURI curi = (CrawlURI) puri; curi.addPersistentDataMapKey(A_FETCH_HISTORY); HashMap latestFetch = new HashMap(); @@ -132,7 +131,7 @@ public class FetchHistoryProcessor extends Processor { @Override - protected boolean shouldProcess(ProcessorURI uri) { + protected boolean shouldProcess(CrawlURI uri) { // TODO evaluate if any pre-eligibility testing should occur return true; } diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java index cf2afb76..c0a3ea1d 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistLoadProcessor.java @@ -1,24 +1,20 @@ -/* PersistLoadProcessor.java - * - * Created on Feb 13, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2007 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.recrawl; @@ -30,7 +26,7 @@ import java.util.logging.Logger; import org.apache.commons.lang.StringUtils; import org.archive.modules.recrawl.PersistLoadProcessor; import org.archive.modules.recrawl.PersistProcessor; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.spring.ConfigFile; import org.archive.spring.ConfigPath; @@ -72,7 +68,7 @@ public class PersistLoadProcessor extends PersistOnlineProcessor { @SuppressWarnings("unchecked") @Override - protected void innerProcess(ProcessorURI curi) throws InterruptedException { + protected void innerProcess(CrawlURI curi) throws InterruptedException { String pkey = persistKeyFor(curi); Map prior = (Map) store.get(pkey); @@ -83,7 +79,7 @@ public class PersistLoadProcessor extends PersistOnlineProcessor { } @Override - protected boolean shouldProcess(ProcessorURI uri) { + protected boolean shouldProcess(CrawlURI uri) { return shouldLoad(uri); } diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java index 77068982..370d0b06 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistLogProcessor.java @@ -1,24 +1,20 @@ -/* PersistLogProcessor.java - * - * Created on Feb 18, 2005 +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * Copyright (C) 2007 Internet Archive. - * - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. - * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. - * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. + * + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.archive.modules.recrawl; @@ -32,7 +28,7 @@ import org.apache.commons.lang.SerializationUtils; import org.archive.checkpointing.Checkpointable; import org.archive.checkpointing.RecoverAction; import org.archive.io.CrawlerJournal; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.spring.ConfigPath; import org.springframework.context.Lifecycle; @@ -95,7 +91,7 @@ implements Checkpointable, Lifecycle { } @Override - protected void innerProcess(ProcessorURI curi) { + protected void innerProcess(CrawlURI curi) { log.writeLine(persistKeyFor(curi), " ", new String(Base64.encodeBase64( SerializationUtils.serialize((Serializable)curi.getData())))); @@ -107,7 +103,7 @@ implements Checkpointable, Lifecycle { } @Override - protected boolean shouldProcess(ProcessorURI uri) { + protected boolean shouldProcess(CrawlURI uri) { return shouldStore(uri); } } \ No newline at end of file diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java index e287c90e..edd5ece4 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistProcessor.java @@ -37,8 +37,8 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.SerializationUtils; import org.archive.bdb.BdbModule; +import org.archive.modules.CrawlURI; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import org.archive.util.FileUtils; import org.archive.util.IoUtils; import org.archive.util.OneLineSimpleLogger; @@ -94,7 +94,7 @@ public abstract class PersistProcessor extends Processor { * @param curi CrawlURI * @return String key */ - public static String persistKeyFor(ProcessorURI curi) { + public static String persistKeyFor(CrawlURI curi) { // use a case-sensitive SURT for uniqueness and sorting benefits return persistKeyFor(curi.getUURI().toString()); } @@ -111,7 +111,7 @@ public abstract class PersistProcessor extends Processor { * @param curi CrawlURI * @return true if state should be stored; false to skip persistence */ - protected boolean shouldStore(ProcessorURI curi) { + protected boolean shouldStore(CrawlURI curi) { // TODO: don't store some codes, such as 304 unchanged? return curi.isSuccess(); } @@ -122,7 +122,7 @@ public abstract class PersistProcessor extends Processor { * @param curi CrawlURI * @return true if state should be loaded; false to skip loading */ - protected boolean shouldLoad(ProcessorURI curi) { + protected boolean shouldLoad(CrawlURI curi) { // TODO: don't load some (prereqs?) return true; } diff --git a/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java b/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java index 66162e75..8065d6db 100644 --- a/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java +++ b/modules/src/main/java/org/archive/modules/recrawl/PersistStoreProcessor.java @@ -19,7 +19,7 @@ package org.archive.modules.recrawl; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Store CrawlURI attributes from latest fetch to persistent storage for * consultation by a later recrawl. @@ -39,12 +39,12 @@ public class PersistStoreProcessor extends PersistOnlineProcessor @SuppressWarnings("unchecked") @Override - protected void innerProcess(ProcessorURI curi) throws InterruptedException { + protected void innerProcess(CrawlURI curi) throws InterruptedException { store.put(persistKeyFor(curi),curi.getPersistentDataMap()); } @Override - protected boolean shouldProcess(ProcessorURI uri) { + protected boolean shouldProcess(CrawlURI uri) { return shouldStore(uri); } } diff --git a/modules/src/main/java/org/archive/modules/seeds/SeedListener.java b/modules/src/main/java/org/archive/modules/seeds/SeedListener.java index 4edc7928..1909f8ba 100644 --- a/modules/src/main/java/org/archive/modules/seeds/SeedListener.java +++ b/modules/src/main/java/org/archive/modules/seeds/SeedListener.java @@ -18,7 +18,7 @@ */ package org.archive.modules.seeds; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; /** * Implemented by components which want notifications of @@ -27,6 +27,6 @@ import org.archive.modules.ProcessorURI; * @author gojomo */ public interface SeedListener { - void addedSeed(final ProcessorURI uuri); + void addedSeed(final CrawlURI uuri); boolean nonseedLine(String line); } diff --git a/modules/src/main/java/org/archive/modules/seeds/SeedModule.java b/modules/src/main/java/org/archive/modules/seeds/SeedModule.java index 56c91970..ce57cea3 100644 --- a/modules/src/main/java/org/archive/modules/seeds/SeedModule.java +++ b/modules/src/main/java/org/archive/modules/seeds/SeedModule.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Set; import org.archive.checkpointing.RecoverAction; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.springframework.beans.factory.annotation.Autowired; public abstract class SeedModule implements Serializable @@ -72,7 +72,7 @@ public abstract class SeedModule implements Serializable this.seedListeners = seedListeners; } - protected void publishAddedSeed(ProcessorURI curi) { + protected void publishAddedSeed(CrawlURI curi) { for (SeedListener l: seedListeners) { l.addedSeed(curi); } @@ -91,7 +91,7 @@ public abstract class SeedModule implements Serializable public abstract void actOn(File f); - public abstract boolean addSeed(final ProcessorURI curi); + public abstract void addSeed(final CrawlURI curi); public abstract void checkpoint(File dir, List actions) throws IOException; diff --git a/modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java b/modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java index 21a8091b..18d94768 100644 --- a/modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java +++ b/modules/src/main/java/org/archive/modules/seeds/TextSeedModule.java @@ -38,7 +38,6 @@ import org.archive.checkpointing.Checkpointable; import org.archive.checkpointing.RecoverAction; import org.archive.io.ReadSource; import org.archive.modules.CrawlURI; -import org.archive.modules.ProcessorURI; import org.archive.modules.SchedulingConstants; import org.archive.net.UURI; import org.archive.net.UURIFactory; @@ -186,7 +185,7 @@ implements ReadSource, * @return true if successful, false if add failed for any reason */ @Override - public synchronized boolean addSeed(final ProcessorURI curi) { + public synchronized void addSeed(final CrawlURI curi) { if(!(textSource instanceof WriteTarget)) { // TODO: do something else to log seed update logger.warning("nowhere to log added seed: "+curi); @@ -201,13 +200,11 @@ implements ReadSource, fw.write(curi.toString()); fw.flush(); fw.close(); - publishAddedSeed(curi); - return true; } catch (IOException e) { DevUtils.warnHandle(e, "problem writing new seed"); } } - return false; + publishAddedSeed(curi); } @Override diff --git a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java index 14c7022e..356fc099 100644 --- a/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/ARCWriterProcessor.java @@ -39,7 +39,7 @@ import org.archive.io.WriterPoolSettings; import org.archive.io.arc.ARCWriter; import org.archive.io.arc.ARCWriterPool; import org.archive.modules.ProcessResult; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.util.ArchiveUtils; /** @@ -82,15 +82,15 @@ public class ARCWriterProcessor extends WriterPoolProcessor { } /** - * Writes a ProcessorURI and its associated data to store file. + * Writes a CrawlURI and its associated data to store file. * * Currently this method understands the following uri types: dns, http, * and https. * - * @param curi ProcessorURI to process. + * @param curi CrawlURI to process. */ - protected ProcessResult innerProcessResult(ProcessorURI puri) { - ProcessorURI curi = (ProcessorURI)puri; + protected ProcessResult innerProcessResult(CrawlURI puri) { + CrawlURI curi = (CrawlURI)puri; long recordLength = getRecordedSize(curi); @@ -113,7 +113,7 @@ public class ARCWriterProcessor extends WriterPoolProcessor { return ProcessResult.PROCEED; } - protected ProcessResult write(ProcessorURI curi, long recordLength, + protected ProcessResult write(CrawlURI curi, long recordLength, InputStream in, String ip) throws IOException { WriterPoolMember writer = getPool().borrowFile(); diff --git a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java index fe6722d0..c1991085 100644 --- a/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/Kw3WriterProcessor.java @@ -44,7 +44,7 @@ import java.util.logging.Logger; import org.archive.io.ReplayInputStream; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; import org.archive.modules.net.ServerCacheUtil; @@ -185,7 +185,7 @@ public class Kw3WriterProcessor extends Processor { public Kw3WriterProcessor() { } - protected boolean shouldProcess(ProcessorURI curi) { + protected boolean shouldProcess(CrawlURI curi) { // Only successful fetches are written. if (!isSuccess(curi)) { return false; @@ -202,7 +202,7 @@ public class Kw3WriterProcessor extends Processor { } - protected void innerProcess(ProcessorURI curi) { + protected void innerProcess(CrawlURI curi) { // Write the MIME-file try { writeMimeFile(curi); @@ -221,7 +221,7 @@ public class Kw3WriterProcessor extends Processor { * * For more on this format, see '?'. */ - protected void writeMimeFile(ProcessorURI curi) throws IOException { + protected void writeMimeFile(CrawlURI curi) throws IOException { ReplayInputStream ris = null; OutputStream out = null; @@ -263,7 +263,7 @@ public class Kw3WriterProcessor extends Processor { * * Example: '/53/www.kb.se/current/6879ad79c0ccf886ee8ca55d80e5d6a1.1169211837' */ - protected OutputStream initOutputStream(ProcessorURI curi) throws IOException { + protected OutputStream initOutputStream(CrawlURI curi) throws IOException { String uri = curi.toString(); int port = curi.getUURI().getPort(); String host = (port == 80 || port <= 0) ? @@ -283,7 +283,7 @@ public class Kw3WriterProcessor extends Processor { return new FastBufferedOutputStream(new FileOutputStream(arcFile)); } - protected void writeArchiveInfoPart(String boundary, ProcessorURI curi, + protected void writeArchiveInfoPart(String boundary, CrawlURI curi, ReplayInputStream ris, OutputStream out) throws IOException { // Get things we need to write in this part @@ -334,7 +334,7 @@ public class Kw3WriterProcessor extends Processor { ris.readHeaderTo(out); } - protected void writeContentPart(String boundary, ProcessorURI curi, + protected void writeContentPart(String boundary, CrawlURI curi, ReplayInputStream ris, OutputStream out) throws IOException { // Get things we need to write in this part @@ -424,7 +424,7 @@ public class Kw3WriterProcessor extends Processor { } } - private String getHostAddress(ProcessorURI curi) { + private String getHostAddress(CrawlURI curi) { CrawlHost h = ServerCacheUtil.getHostFor(serverCache, curi.getUURI()); if (h == null) { throw new NullPointerException("Crawlhost is null for " + curi + " " + diff --git a/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java index 420504a9..263dacc9 100644 --- a/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/MirrorWriterProcessor.java @@ -47,8 +47,8 @@ import javax.management.AttributeNotFoundException; import org.apache.commons.io.IOUtils; import org.archive.io.RecordingInputStream; import org.archive.io.ReplayInputStream; +import org.archive.modules.CrawlURI; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import org.archive.net.UURI; import org.archive.util.IoUtils; @@ -333,13 +333,13 @@ public class MirrorWriterProcessor extends Processor { @Override - protected boolean shouldProcess(ProcessorURI curi) { + protected boolean shouldProcess(CrawlURI curi) { return isSuccess(curi); } @Override - protected void innerProcess(ProcessorURI curi) { + protected void innerProcess(CrawlURI curi) { UURI uuri = curi.getUURI(); // Current URI. // Only http and https schemes are supported. @@ -482,7 +482,7 @@ public class MirrorWriterProcessor extends Processor { @throws IOException if a non-directory file exists with the same path as a needed directory */ - private URIToFileReturn uriToFile(String baseDir, ProcessorURI curi) + private URIToFileReturn uriToFile(String baseDir, CrawlURI curi) throws AttributeNotFoundException, IOException { UURI uuri = curi.getUURI(); // Current URI. String host = null; @@ -632,7 +632,7 @@ public class MirrorWriterProcessor extends Processor { @throws IOException if a non-directory file exists with the same path as a needed directory */ - private URIToFileReturn uriToFile(ProcessorURI curi, String host, int port, + private URIToFileReturn uriToFile(CrawlURI curi, String host, int port, String uriPath, String query, String suffix, String baseDir, int maxSegLen, int maxPathLen, boolean caseSensitive, String dirFile, Map characterMap, String dotBegin, String dotEnd, @@ -773,7 +773,7 @@ public class MirrorWriterProcessor extends Processor { protected static final int EXISTS_CASE_INSENSITIVE_MATCH = 3; /** The URI, for logging and error reporting.*/ - protected ProcessorURI curi; + protected CrawlURI curi; /** The main part of this segment. @@ -808,7 +808,7 @@ public class MirrorWriterProcessor extends Processor { @throws IllegalArgumentException if maxSegLen is too small */ - PathSegment(int maxSegLen, boolean caseSensitive, ProcessorURI curi) { + PathSegment(int maxSegLen, boolean caseSensitive, CrawlURI curi) { if (maxSegLen < 2) { throw new IllegalArgumentException("maxSegLen: " + maxSegLen); } @@ -937,7 +937,7 @@ public class MirrorWriterProcessor extends Processor { maxSegLen is too small. */ DirSegment(String uriPath, int beginIndex, int endIndex, int maxSegLen, - boolean caseSensitive, ProcessorURI curi, Map characterMap, + boolean caseSensitive, CrawlURI curi, Map characterMap, String dotBegin, String dotEnd, Set underscoreSet) { super(maxSegLen, caseSensitive, curi); mainPart = new LumpyString(uriPath, beginIndex, endIndex, @@ -1126,7 +1126,7 @@ public class MirrorWriterProcessor extends Processor { maxSegLen is too small. */ EndSegment(String uriPath, int beginIndex, int endIndex, int maxSegLen, - boolean caseSensitive, ProcessorURI curi, Map characterMap, + boolean caseSensitive, CrawlURI curi, Map characterMap, String dotBegin, String query, String suffix, int maxPathLen, boolean suffixAtEnd) { super(maxSegLen - 1, caseSensitive, curi); diff --git a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java index ec080bc7..c4772c80 100644 --- a/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WARCWriterProcessor.java @@ -73,7 +73,6 @@ import org.archive.io.warc.WARCWriterPool; import org.archive.modules.CrawlMetadata; import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; -import org.archive.modules.ProcessorURI; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; import org.archive.modules.extractor.Link; import org.archive.uid.GeneratorFactory; @@ -169,17 +168,17 @@ public class WARCWriterProcessor extends WriterPoolProcessor { } /** - * Writes a ProcessorURI and its associated data to store file. + * Writes a CrawlURI and its associated data to store file. * * Currently this method understands the following uri types: dns, http, and * https. * - * @param curi ProcessorURI to process. + * @param curi CrawlURI to process. * */ @Override - protected ProcessResult innerProcessResult(ProcessorURI puri) { - ProcessorURI curi = (ProcessorURI)puri; + protected ProcessResult innerProcessResult(CrawlURI puri) { + CrawlURI curi = (CrawlURI)puri; String scheme = curi.getUURI().getScheme().toLowerCase(); try { if (shouldWrite(curi)) { @@ -197,7 +196,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { } protected ProcessResult write(final String lowerCaseScheme, - final ProcessorURI curi) + final CrawlURI curi) throws IOException { WriterPoolMember writer = getPool().borrowFile(); long position = writer.getPosition(); @@ -304,7 +303,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { protected URI writeRequest(final WARCWriter w, final String timestamp, final String mimetype, - final URI baseid, final ProcessorURI curi, + final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { final URI uid = qualifyRecordID(baseid, TYPE, REQUEST); @@ -322,7 +321,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { protected URI writeResponse(final WARCWriter w, final String timestamp, final String mimetype, - final URI baseid, final ProcessorURI curi, + final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { ReplayInputStream ris = @@ -339,7 +338,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { protected URI writeResource(final WARCWriter w, final String timestamp, final String mimetype, - final URI baseid, final ProcessorURI curi, + final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { ReplayInputStream ris = curi.getRecorder().getRecordedInput().getReplayInputStream(); @@ -355,7 +354,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { protected URI writeRevisitDigest(final WARCWriter w, final String timestamp, final String mimetype, - final URI baseid, final ProcessorURI curi, + final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { long revisedLength = curi.getRecorder().getRecordedInput().getContentBegin(); @@ -380,7 +379,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { protected URI writeRevisitNotModified(final WARCWriter w, final String timestamp, - final URI baseid, final ProcessorURI puri, + final URI baseid, final CrawlURI puri, final ANVLRecord namedFields) throws IOException { CrawlURI curi = (CrawlURI) puri; @@ -425,7 +424,7 @@ public class WARCWriterProcessor extends WriterPoolProcessor { protected URI writeMetadata(final WARCWriter w, final String timestamp, - final URI baseid, final ProcessorURI curi, + final URI baseid, final CrawlURI curi, final ANVLRecord namedFields) throws IOException { final URI uid = qualifyRecordID(baseid, TYPE, METADATA); diff --git a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java index 4c0ab507..d4e04c4c 100644 --- a/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java +++ b/modules/src/main/java/org/archive/modules/writer/WriterPoolProcessor.java @@ -39,9 +39,9 @@ import org.archive.io.WriterPool; import org.archive.io.WriterPoolMember; import org.archive.io.WriterPoolSettings; import org.archive.modules.CrawlMetadata; +import org.archive.modules.CrawlURI; import org.archive.modules.ProcessResult; import org.archive.modules.Processor; -import org.archive.modules.ProcessorURI; import org.archive.modules.deciderules.recrawl.IdenticalDigestDecideRule; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; @@ -155,7 +155,7 @@ implements Lifecycle { } /** - * ProcessorURI annotation indicating no record was written. + * CrawlURI annotation indicating no record was written. */ protected static final String ANNOTATION_UNWRITTEN = "unwritten"; @@ -279,13 +279,13 @@ implements Lifecycle { } /** - * Whether the given ProcessorURI should be written to archive files. - * Annotates ProcessorURI with a reason for any negative answer. + * Whether the given CrawlURI should be written to archive files. + * Annotates CrawlURI with a reason for any negative answer. * - * @param curi ProcessorURI + * @param curi CrawlURI * @return true if URI should be written; false otherwise */ - protected boolean shouldWrite(ProcessorURI curi) { + protected boolean shouldWrite(CrawlURI curi) { if (getSkipIdenticalDigests() && IdenticalDigestDecideRule.hasIdenticalDigest(curi)) { curi.getAnnotations().add(ANNOTATION_UNWRITTEN @@ -295,7 +295,7 @@ implements Lifecycle { boolean retVal; String scheme = curi.getUURI().getScheme().toLowerCase(); - // TODO: possibly move this sort of isSuccess() test into ProcessorURI + // TODO: possibly move this sort of isSuccess() test into CrawlURI if (scheme.equals("dns")) { retVal = curi.getFetchStatus() == S_DNS_SUCCESS; } else if (scheme.equals("http") || scheme.equals("https")) { @@ -321,10 +321,10 @@ implements Lifecycle { * Return IP address of given URI suitable for recording (as in a * classic ARC 5-field header line). * - * @param curi ProcessorURI + * @param curi CrawlURI * @return String of IP address */ - protected String getHostAddress(ProcessorURI curi) { + protected String getHostAddress(CrawlURI curi) { // special handling for DNS URIs: want address of DNS server if (curi.getUURI().getScheme().toLowerCase().equals("dns")) { return (String)curi.getData().get(A_DNS_SERVER_IP_LABEL); @@ -450,19 +450,19 @@ implements Lifecycle { } @Override - protected void innerProcess(ProcessorURI puri) { + protected void innerProcess(CrawlURI puri) { throw new AssertionError(); } @Override - protected abstract ProcessResult innerProcessResult(ProcessorURI uri); + protected abstract ProcessResult innerProcessResult(CrawlURI uri); - protected boolean shouldProcess(ProcessorURI uri) { - if (!(uri instanceof ProcessorURI)) { + protected boolean shouldProcess(CrawlURI uri) { + if (!(uri instanceof CrawlURI)) { return false; } - ProcessorURI curi = (ProcessorURI)uri; + CrawlURI curi = (CrawlURI)uri; // If failure, or we haven't fetched the resource yet, return if (curi.getFetchStatus() <= 0) { return false; diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java index 34495d03..f6775014 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorCSSTest.java @@ -22,7 +22,7 @@ package org.archive.modules.extractor; import java.util.Collection; import java.util.Collections; -import org.archive.modules.DefaultProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.modules.extractor.Extractor; import org.archive.modules.extractor.ExtractorCSS; import org.archive.modules.extractor.Hop; @@ -75,11 +75,11 @@ public class ExtractorCSSTest extends StringExtractorTestBase { protected Collection makeData(String content, String uri) throws Exception { UURI src = UURIFactory.getInstance("http://www.archive.org/start/"); - DefaultProcessorURI euri = new DefaultProcessorURI(src, NAVLINK_MISC); + CrawlURI euri = new CrawlURI(src, null, null, NAVLINK_MISC); Recorder recorder = createRecorder(content); euri.setContentType("text/css"); euri.setRecorder(recorder); - euri.setContentLength(content.length()); + euri.setContentSize(content.length()); UURI dest = UURIFactory.getInstance(uri); Link link = new Link(src, dest, EMBED_MISC, Hop.EMBED); diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 5d83b643..b46523f5 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -20,13 +20,14 @@ package org.archive.modules.extractor; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.Predicate; import org.apache.commons.httpclient.URIException; -import org.archive.modules.DefaultProcessorURI; +import org.archive.modules.CrawlURI; import org.archive.net.UURI; import org.archive.net.UURIFactory; import org.archive.util.Recorder; @@ -86,12 +87,12 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { throws Exception { List result = new ArrayList(); UURI src = UURIFactory.getInstance("http://www.archive.org/start/"); - DefaultProcessorURI euri = new DefaultProcessorURI(src, + CrawlURI euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC); Recorder recorder = createRecorder(content); euri.setContentType("text/html"); euri.setRecorder(recorder); - euri.setContentLength(content.length()); + euri.setContentSize(content.length()); UURI dest = UURIFactory.getInstance(destURI); LinkContext context = determineContext(content); @@ -99,11 +100,11 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { Link link = new Link(src, dest, context, hop); result.add(new TestData(euri, link)); - euri = new DefaultProcessorURI(src, LinkContext.NAVLINK_MISC); + euri = new CrawlURI(src, null, null, LinkContext.NAVLINK_MISC); recorder = createRecorder(content); euri.setContentType("application/xhtml"); euri.setRecorder(recorder); - euri.setContentLength(content.length()); + euri.setContentSize(content.length()); result.add(new TestData(euri, link)); return result; @@ -152,8 +153,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { * @throws URIException */ protected void expectSingleLink(String expected, CharSequence source) throws URIException { - DefaultProcessorURI puri = new DefaultProcessorURI(UURIFactory - .getInstance("http://www.example.com"), null); + CrawlURI puri = new CrawlURI(UURIFactory + .getInstance("http://www.example.com")); ExtractorHTML extractor = (ExtractorHTML)makeExtractor(); extractor.extract(puri, source); Link[] links = puri.getOutLinks().toArray(new Link[0]); @@ -170,8 +171,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { * http://webteam.archive.org/jira/browse/HER-1280 */ public void testOnlyExtractFormGets() throws URIException { - DefaultProcessorURI puri = new DefaultProcessorURI(UURIFactory - .getInstance("http://www.example.com"),null); + CrawlURI puri = new CrawlURI(UURIFactory + .getInstance("http://www.example.com")); CharSequence cs = "
"+ " "+ @@ -188,8 +189,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { * Test detection, respect of meta robots nofollow directive */ public void testMetaRobots() throws URIException { - DefaultProcessorURI puri = new DefaultProcessorURI(UURIFactory - .getInstance("http://www.example.com"),null); + CrawlURI puri = new CrawlURI(UURIFactory + .getInstance("http://www.example.com")); CharSequence cs = "Blah Blah "+ ""+ @@ -212,8 +213,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { * @throws URIException */ public void testBadRelativeLinks() throws URIException { - DefaultProcessorURI curi = new DefaultProcessorURI(UURIFactory - .getInstance("http://www.example.com"), null); + CrawlURI curi = new CrawlURI(UURIFactory + .getInstance("http://www.example.com")); CharSequence cs = "" + ""; ExtractorHTML extractor = (ExtractorHTML)makeExtractor(); @@ -244,8 +245,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { * [HER-1524] speculativeFixup in ExtractorJS should maintain URL scheme */ public void testSpeculativeLinkExtraction() throws URIException { - DefaultProcessorURI curi = new DefaultProcessorURI(UURIFactory - .getInstance("https://www.example.com"), null); + CrawlURI curi = new CrawlURI(UURIFactory + .getInstance("https://www.example.com")); CharSequence cs = "