mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-10 07:52:46 +00:00
Cleanup, refactoring
* (many) update license notice to Apache where appropriate eliminate ProcessorURI, DefaultProcessorURI in favor of CrawlURI (now in modules package) other comment/warning/unused code cleanup
This commit is contained in:
gojomo committed
1 parent
c04c5dbdbc
commit
16538f175c
92 files changed
+1315
-2118
No files matched your search
@@ -1,25 +1,22 @@
|
||||
/* Scoper
|
||||
*
|
||||
* Created on Jun 6, 2005
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Copyright (C) 2005 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.framework;
|
||||
|
||||
import java.util.logging.FileHandler;
|
||||
@@ -80,7 +77,7 @@ public abstract class Scoper extends Processor implements Lifecycle {
|
||||
return this.scope;
|
||||
}
|
||||
@Autowired
|
||||
public void setDecideRule(DecideRule scope) {
|
||||
public void setScope(DecideRule scope) {
|
||||
this.scope = scope;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,24 @@
|
||||
package org.archive.crawler.frontier;
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.frontier;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.Serializable;
|
||||
@@ -14,7 +33,6 @@ import org.archive.crawler.framework.Frontier;
|
||||
import org.archive.crawler.frontier.precedence.PrecedenceProvider;
|
||||
import org.archive.crawler.frontier.precedence.SimplePrecedenceProvider;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.ProcessorURI;
|
||||
import org.archive.modules.fetcher.FetchStats;
|
||||
import org.archive.modules.fetcher.FetchStats.Stage;
|
||||
import org.archive.util.ArchiveUtils;
|
||||
@@ -665,9 +683,9 @@ public abstract class WorkQueue implements Frontier.FrontierGroup,
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.modules.fetcher.FetchStats.HasFetchStats#tally(org.archive.modules.ProcessorURI, org.archive.modules.fetcher.FetchStats.Stage)
|
||||
* @see org.archive.modules.fetcher.FetchStats.HasFetchStats#tally(org.archive.modules.CrawlURI, org.archive.modules.fetcher.FetchStats.Stage)
|
||||
*/
|
||||
public void tally(ProcessorURI curi, Stage stage) {
|
||||
public void tally(CrawlURI curi, Stage stage) {
|
||||
substats.tally(curi, stage);
|
||||
precedenceProvider.tally(curi, stage);
|
||||
}
|
||||
|
||||
@@ -1,24 +1,20 @@
|
||||
/* QuotaEnforcer
|
||||
*
|
||||
* Created on Nov 4, 2005
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Copyright (C) 2005 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.crawler.prefetch;
|
||||
|
||||
@@ -31,7 +27,6 @@ import org.archive.crawler.framework.Frontier;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.ProcessResult;
|
||||
import org.archive.modules.Processor;
|
||||
import org.archive.modules.ProcessorURI;
|
||||
import org.archive.modules.fetcher.FetchStats;
|
||||
import org.archive.modules.net.CrawlHost;
|
||||
import org.archive.modules.net.CrawlServer;
|
||||
@@ -310,15 +305,15 @@ public class QuotaEnforcer extends Processor {
|
||||
this.frontier = frontier;
|
||||
}
|
||||
|
||||
protected boolean shouldProcess(ProcessorURI puri) {
|
||||
protected boolean shouldProcess(CrawlURI puri) {
|
||||
return puri instanceof CrawlURI;
|
||||
}
|
||||
|
||||
protected void innerProcess(ProcessorURI puri) {
|
||||
protected void innerProcess(CrawlURI puri) {
|
||||
throw new AssertionError();
|
||||
}
|
||||
|
||||
protected ProcessResult innerProcessResult(ProcessorURI puri) {
|
||||
protected ProcessResult innerProcessResult(CrawlURI puri) {
|
||||
CrawlURI curi = (CrawlURI)puri;
|
||||
final CrawlServer server = ServerCacheUtil.getServerFor(serverCache,
|
||||
curi.getUURI());
|
||||
|
||||
@@ -1,41 +1,35 @@
|
||||
/* RuntimeLimitEnforcer
|
||||
*
|
||||
* Created on July 7, 2006
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Copyright (C) 2005 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.archive.crawler.prefetch;
|
||||
|
||||
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_RUNTIME_LIMIT;
|
||||
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.*;
|
||||
|
||||
import org.archive.crawler.framework.CrawlController;
|
||||
import org.archive.crawler.framework.CrawlStatus;
|
||||
import org.archive.crawler.reporting.StatisticsTracker;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.ProcessResult;
|
||||
import org.archive.modules.Processor;
|
||||
import org.archive.modules.ProcessorURI;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
|
||||
|
||||
@@ -160,18 +154,18 @@ public class RuntimeLimitEnforcer extends Processor {
|
||||
|
||||
|
||||
@Override
|
||||
protected boolean shouldProcess(ProcessorURI puri) {
|
||||
protected boolean shouldProcess(CrawlURI puri) {
|
||||
return puri instanceof CrawlURI;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void innerProcess(ProcessorURI curi) {
|
||||
protected void innerProcess(CrawlURI curi) {
|
||||
throw new AssertionError();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ProcessResult innerProcessResult(ProcessorURI curi)
|
||||
protected ProcessResult innerProcessResult(CrawlURI curi)
|
||||
throws InterruptedException {
|
||||
CrawlController controller = getCrawlController();
|
||||
StatisticsTracker stats = getStatisticsTracker();
|
||||
|
||||
@@ -1,25 +1,22 @@
|
||||
/* CrawlMapper
|
||||
*
|
||||
* Created on Sep 30, 2005
|
||||
/*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Copyright (C) 2005 Internet Archive.
|
||||
*
|
||||
* This file is part of the Heritrix web crawler (crawler.archive.org).
|
||||
*
|
||||
* Heritrix is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser Public License as published by
|
||||
* the Free Software Foundation; either version 2.1 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* Heritrix is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser Public License
|
||||
* along with Heritrix; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
* Licensed to the Internet Archive (IA) by one or more individual
|
||||
* contributors.
|
||||
*
|
||||
* The IA licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.archive.crawler.processor;
|
||||
|
||||
import static org.archive.modules.fetcher.FetchStatusCodes.S_BLOCKED_BY_CUSTOM_PROCESSOR;
|
||||
@@ -35,7 +32,6 @@ import java.util.Iterator;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.ProcessResult;
|
||||
import org.archive.modules.Processor;
|
||||
import org.archive.modules.ProcessorURI;
|
||||
import org.archive.modules.deciderules.AcceptDecideRule;
|
||||
import org.archive.modules.deciderules.DecideResult;
|
||||
import org.archive.modules.deciderules.DecideRule;
|
||||
@@ -182,17 +178,17 @@ public abstract class CrawlMapper extends Processor implements Lifecycle {
|
||||
|
||||
|
||||
@Override
|
||||
protected boolean shouldProcess(ProcessorURI puri) {
|
||||
protected boolean shouldProcess(CrawlURI puri) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void innerProcess(ProcessorURI puri) {
|
||||
protected void innerProcess(CrawlURI puri) {
|
||||
throw new AssertionError();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ProcessResult innerProcessResult(ProcessorURI puri) {
|
||||
protected ProcessResult innerProcessResult(CrawlURI puri) {
|
||||
CrawlURI curi = (CrawlURI)puri;
|
||||
String nowGeneration =
|
||||
ArchiveUtils.get14DigitDate().substring(
|
||||
|
||||
Reference in new issue
Block a user