mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-07-17 13:07:22 +00:00
gojomo
8d1083922d
* .classpath, pom.xml, org/apache/commons/pool/impl/*
remove references to and patched classes based on commons-pool
* WriterPool.java
move instance creation/round-robin-pool/dispose into this class
close (rather then recycle) extra writers when crawl slows (via tracking time of last need/rollover)
* WriterPoolMember.java
add isOversize() for external check of ready-to-rollover file
avoid rollover before every record (so that set of related WARC records aren't split between files)
* ARCWriterPool.java, WARCWriterPool.java
remove BasePoolableObjectFactory reference, implement class-appropriate makeWriter()
* ARCWriterPoolTest.java
adjust for new never-timeout behavior
* ARCWriterPoolProcessor.java WARCWriterPoolProcessor.java
adjust constructor calls
* WriterPoolProcessor.java
replace poolMaxWaitMs (which caused eventual complete timeout from waiting for writer) with maxWaitForIdleMs, a much smaller value which only controls how long a thread waits for an reusable writer before considering creating a new one. (If creation isn't allowed, threads will now wait indefinitely for a writer to become available.)
329 lines
12 KiB
XML
329 lines
12 KiB
XML
<?xml version="1.0"?>
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
|
<parent>
|
|
<groupId>org.archive</groupId>
|
|
<artifactId>heritrix</artifactId>
|
|
<version>3.1.1-SNAPSHOT</version>
|
|
</parent>
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<groupId>org.archive.heritrix</groupId>
|
|
<artifactId>heritrix-commons</artifactId>
|
|
<packaging>jar</packaging>
|
|
<name>Heritrix 3: 'commons' subproject (utility classes)</name>
|
|
<description>
|
|
The Archive Commons Code Libraries project contains general Java utility
|
|
libraries, as used by the Heritrix crawler and other projects.
|
|
</description>
|
|
|
|
<repositories>
|
|
<repository>
|
|
<releases>
|
|
<enabled>true</enabled>
|
|
<updatePolicy>always</updatePolicy>
|
|
<checksumPolicy>warn</checksumPolicy>
|
|
</releases>
|
|
<snapshots>
|
|
<enabled>true</enabled>
|
|
<updatePolicy>never</updatePolicy>
|
|
<checksumPolicy>fail</checksumPolicy>
|
|
</snapshots>
|
|
<id>oracleReleases</id>
|
|
<name>Oracle Released Java Packages</name>
|
|
<url>http://download.oracle.com/maven</url>
|
|
<layout>default</layout>
|
|
</repository>
|
|
|
|
<repository>
|
|
<releases>
|
|
<enabled>true</enabled>
|
|
<updatePolicy>daily</updatePolicy>
|
|
<checksumPolicy>warn</checksumPolicy>
|
|
</releases>
|
|
<snapshots>
|
|
<enabled>true</enabled>
|
|
<updatePolicy>never</updatePolicy>
|
|
<checksumPolicy>fail</checksumPolicy>
|
|
</snapshots>
|
|
<id>internetarchive</id>
|
|
<name>Internet Archive Maven Repository</name>
|
|
<url>http://builds.archive.org:8080/maven2</url>
|
|
<layout>default</layout>
|
|
</repository>
|
|
</repositories>
|
|
|
|
<dependencies>
|
|
|
|
|
|
<!--
|
|
|
|
This project requires "overlays". There are custom patches we
|
|
apply to existing jar files from other projects (Apache). Those
|
|
projects would not/could not accept our patches, so
|
|
|
|
Maven doesn't have any sort of notion for this. It should be
|
|
a simple matter of ordering classpath elements, but alas.
|
|
|
|
We wanted to avoid checking in the complete source code for these
|
|
other projects into our source control, so it would be obvious
|
|
which classes we actually altered to suit our needs.
|
|
|
|
I tried a number of things to make overlay jars work in maven.
|
|
Here's what finally worked:
|
|
|
|
I manually explode the jar files, then manually delete the few class
|
|
files that we overlay (including any inner class files). I then
|
|
rejar the class files minus the ones we want to generate at
|
|
compile-time. I use a name archive-overlay-X-V.jar, where X
|
|
is the original artifact name, and V is the original version.
|
|
|
|
I then manually install these jars into our maven repository.
|
|
So the below dependencies are these strange, surgically altered
|
|
jars from other projects.
|
|
|
|
It's ugly but it works. If you make any additional changes to
|
|
these projects, or heavens forfend you actually want to upgrade
|
|
them to some later version, you will have to repeat the
|
|
explode/delete/rejar/reinstall manual process.
|
|
-->
|
|
|
|
<dependency>
|
|
<groupId>org.archive.overlays</groupId>
|
|
<artifactId>archive-overlay-commons-httpclient</artifactId>
|
|
<version>3.1</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
|
|
<!-- These are normal dependencies. -->
|
|
<dependency>
|
|
<groupId>com.sleepycat</groupId>
|
|
<artifactId>je</artifactId>
|
|
<version>4.0.71</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-lang</groupId>
|
|
<artifactId>commons-lang</artifactId>
|
|
<version>2.3</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-logging</groupId>
|
|
<artifactId>commons-logging</artifactId>
|
|
<version>1.0.4</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-net</groupId>
|
|
<artifactId>commons-net</artifactId>
|
|
<version>2.0</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-io</groupId>
|
|
<artifactId>commons-io</artifactId>
|
|
<version>1.4</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-codec</groupId>
|
|
<artifactId>commons-codec</artifactId>
|
|
<version>1.3</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-collections</groupId>
|
|
<artifactId>commons-collections</artifactId>
|
|
<version>3.1</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>commons-cli</groupId>
|
|
<artifactId>commons-cli</artifactId>
|
|
<version>1.1</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>net.htmlparser.jericho</groupId>
|
|
<artifactId>jericho-html</artifactId>
|
|
<version>2.6.1</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.dnsjava</groupId>
|
|
<artifactId>dnsjava</artifactId>
|
|
<version>2.0.3</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<!--
|
|
<dependency>
|
|
<groupId>tomcat</groupId>
|
|
<artifactId>servlet</artifactId>
|
|
<version>4.1.34</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
-->
|
|
<dependency>
|
|
<groupId>poi</groupId>
|
|
<artifactId>poi</artifactId>
|
|
<version>2.5.1</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>poi</groupId>
|
|
<artifactId>poi-scratchpad</artifactId>
|
|
<version>2.5.1-final-20040804</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>itext</groupId>
|
|
<artifactId>itext</artifactId>
|
|
<version>1.3</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
<version>3.8.2</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>fastutil</groupId>
|
|
<artifactId>fastutil</artifactId>
|
|
<version>5.0.7</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.gnu.inet</groupId>
|
|
<artifactId>libidn</artifactId>
|
|
<version>0.6.5</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>net.java.dev.jets3t</groupId>
|
|
<artifactId>jets3t</artifactId>
|
|
<version>0.5.0</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>it.unimi.dsi</groupId>
|
|
<artifactId>mg4j</artifactId>
|
|
<version>1.0.1</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.anotherbigidea</groupId>
|
|
<artifactId>javaswf</artifactId>
|
|
<version>CVS-SNAPSHOT-1</version>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.springframework</groupId>
|
|
<artifactId>spring-core</artifactId>
|
|
<version>2.5.6</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.springframework</groupId>
|
|
<artifactId>spring-beans</artifactId>
|
|
<version>2.5.6</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.springframework</groupId>
|
|
<artifactId>spring-context</artifactId>
|
|
<version>2.5.6</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>joda-time</groupId>
|
|
<artifactId>joda-time</artifactId>
|
|
<version>1.6</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.json</groupId>
|
|
<artifactId>json</artifactId>
|
|
<version>20090211</version>
|
|
</dependency>
|
|
</dependencies>
|
|
<build>
|
|
<resources>
|
|
<resource>
|
|
<directory>src/main/resources</directory>
|
|
<filtering>true</filtering>
|
|
</resource>
|
|
</resources>
|
|
|
|
<plugins>
|
|
|
|
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-resources-plugin</artifactId>
|
|
<configuration>
|
|
<encoding>UTF-8</encoding>
|
|
</configuration>
|
|
</plugin>
|
|
|
|
<plugin>
|
|
<artifactId>maven-antrun-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>generate-timestamp-properties-file</id>
|
|
<phase>process-resources</phase>
|
|
<goals><goal>run</goal></goals>
|
|
<configuration>
|
|
<tasks>
|
|
<tstamp>
|
|
<format property="timestamp"
|
|
pattern="yyyyMMdd.HHmmss"/>
|
|
</tstamp>
|
|
<delete
|
|
file="target/classes/org/archive/util/timestamp.txt"
|
|
failonerror="false"/>
|
|
<echo message="timestamp=${timestamp}"
|
|
file="target/classes/org/archive/util/timestamp.txt"/>
|
|
</tasks>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-compiler-plugin</artifactId>
|
|
<configuration>
|
|
<source>1.5</source>
|
|
<target>1.5</target>
|
|
</configuration>
|
|
</plugin>
|
|
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-surefire-plugin</artifactId>
|
|
<configuration>
|
|
<!--
|
|
There was a unit test, SinkHandlerTest, that required
|
|
useSystemClassLoader=true in order to work.
|
|
|
|
SinkHandlerTest creates a custom LogHandler, and the
|
|
java.util.logging system mandates that LogHandlers are
|
|
loaded by the system class loader, not the current context
|
|
class loader.
|
|
|
|
However, using the systemClassLoader means that we inherit
|
|
maven's CLASSPATH while running our test code. This is a
|
|
problem since maven uses an earlier version of
|
|
commons-lang than we do.
|
|
|
|
So I disabled the SinkHandler test, and set this back to false.
|
|
-->
|
|
<useSystemClassLoader>false</useSystemClassLoader>
|
|
<excludes>
|
|
<exclude>**/Test*.java</exclude>
|
|
</excludes>
|
|
<argLine>-Xmx1024m</argLine>
|
|
</configuration>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
<properties>
|
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
|
</properties>
|
|
</project>
|