mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-11-08 23:13:28 +00:00
Alex Osborne
19c24b9006
This enables crawl configuration files to use Spring's [Groovy Bean Definition DSL] as an optional alternative to Spring XML. It uses the same bean configuration model but the syntax is more terse and human-readable. No more need for `&` in seed URLs. :-)
```groovy
checkpointService(CheckpointService) {
checkpointIntervalMinutes = 15
checkpointsDir = 'checkpoints'
forgetAllButLatest = true
}
```
It also enables some powerful scripting capabilities. For example, defining a custom DecideRule directly in the crawl scope:
```groovy
scope(DecideRuleSequence) {
rules = [
new RejectDecideRule(),
// ACCEPT everything linked from a .pdf file
new PredicatedDecideRule() {
boolean evaluate(CrawlURI uri) {
return uri.via?.path?.endsWith(".pdf")
}
},
// ...
]
}
```
The main downsides are defining nested inner beans can be a bit awkward, some of the errors can be cryptic, and you can't just manipulate the config files with an XML parser.
This commit includes a Groovy version of the default crawl profile for reference, but doesn't expose a way to use it in the UI yet. For now, you need to manually create a `crawler-beans.groovy` file in your job directory.
[Groovy Bean Definition DSL]: https://docs.spring.io/spring-framework/reference/core/beans/basics.html#beans-factory-groovy
120 lines
3.6 KiB
XML
120 lines
3.6 KiB
XML
<?xml version="1.0"?>
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
|
<parent>
|
|
<groupId>org.archive</groupId>
|
|
<artifactId>heritrix</artifactId>
|
|
<version>3.6.1-SNAPSHOT</version>
|
|
</parent>
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<groupId>org.archive.heritrix</groupId>
|
|
<artifactId>heritrix-modules</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Heritrix 3: 'modules' subproject (reusable components)</name>
|
|
<description>
|
|
This project contains some of the configurable modules used within the
|
|
Heritrix application to crawl the web. The modules in this project can
|
|
be used in applications other than Heritrix, however.
|
|
</description>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.archive.heritrix</groupId>
|
|
<artifactId>heritrix-commons</artifactId>
|
|
<version>${project.version}</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.beanshell</groupId>
|
|
<artifactId>bsh</artifactId>
|
|
<version>2.0b5</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.groovy</groupId>
|
|
<artifactId>groovy-jsr223</artifactId>
|
|
<version>${groovy.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.groovy</groupId>
|
|
<artifactId>groovy-templates</artifactId>
|
|
<version>${groovy.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-server</artifactId>
|
|
<version>${jetty.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-security</artifactId>
|
|
<version>${jetty.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-servlet</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.eclipse.jetty</groupId>
|
|
<artifactId>jetty-proxy</artifactId>
|
|
<version>${jetty.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
<version>${slf4j.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-jdk14</artifactId>
|
|
<version>${slf4j.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.ftpserver</groupId>
|
|
<artifactId>ftpserver-core</artifactId>
|
|
<version>1.2.0</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.github.crawler-commons</groupId>
|
|
<artifactId>crawler-commons</artifactId>
|
|
<version>1.4</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.github.bbottema</groupId>
|
|
<artifactId>java-socks-proxy-server</artifactId>
|
|
<version>4.1.2</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.pdfbox</groupId>
|
|
<artifactId>pdfbox</artifactId>
|
|
<version>3.0.3</version>
|
|
</dependency>
|
|
</dependencies>
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-surefire-plugin</artifactId>
|
|
<version>2.22.2</version>
|
|
<configuration>
|
|
<excludes>
|
|
<exclude>**/TestAll.java</exclude>
|
|
<!-- exclude FetchHTTPTests because its not meant to be invoked directly and newer
|
|
versions of surefire are more lax about the filenames they pickup -->
|
|
<exclude>**/*Tests.java</exclude>
|
|
</excludes>
|
|
</configuration>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
<properties>
|
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
|
</properties>
|
|
</project>
|