This class provides buffering for output streams, but it does so with - * purposes and an internal logic that are radically different from the ones - * adopted in {@link java.io.BufferedOutputStream}. - * - *
All methods are unsychronised. Moreover, - * it is guaranteed that all writes performed by this class will be - * multiples of the given buffer size. If, for instance, you use the - * default buffer size, writes will be performed on the underlying input stream - * in multiples of 16384 bytes. This is very important on operating systems - * that optimise disk reads on disk block boundaries. +/** + * FastBufferedOutputStream that accepts a passed-in buffer (avoiding + * reallocation). */ - -public class RecyclingFastBufferedOutputStream extends OutputStream { - - /** The default size of the internal buffer in bytes (16Ki). */ - public final static int DEFAULT_BUFFER_SIZE = 16 * 1024; - - /** The internal buffer. */ - protected byte buffer[]; - - /** The current position in the buffer. */ - protected int pos; - - /** The number of buffer bytes available starting from {@link #pos}. */ - protected int avail; - - /** The underlying output stream. */ - protected OutputStream os; - - /** Creates a new fast buffered output stream by wrapping a given output stream, using a given buffer - * - * @param os an output stream to wrap. - * @param buffer buffer to use internally. - */ - +public class RecyclingFastBufferedOutputStream extends FastBufferedOutputStream { public RecyclingFastBufferedOutputStream( final OutputStream os, final byte[] buffer ) { - this.os = os; + super(os); this.buffer = buffer; avail = buffer.length; } - - /** Creates a new fast buffered output stream by wrapping a given output stream with a given buffer size. - * - * @param os an output stream to wrap. - * @param bufSize the size in bytes of the internal buffer. - */ - - public RecyclingFastBufferedOutputStream( final OutputStream os, final int bufSize ) { - this(os, new byte [ bufSize]); - } - - /** Creates a new fast buffered ouptut stream by wrapping a given output stream with a buffer of {@link #DEFAULT_BUFFER_SIZE} bytes. - * - * @param os an output stream to wrap. - */ - public RecyclingFastBufferedOutputStream( final OutputStream os ) { - this( os, DEFAULT_BUFFER_SIZE ); - } - - private void dumpBufferIfFull() throws IOException { - if ( avail == 0 ) { - os.write( buffer ); - pos = 0; - avail = buffer.length; - } - } - - public void write( final int b ) throws IOException { - avail--; - buffer[ pos++ ] = (byte)b; - dumpBufferIfFull(); - } - - - public void write( final byte b[], int offset, int length ) throws IOException { - if ( length <= avail ) { - System.arraycopy( b, offset, buffer, pos, length ); - pos += length; - avail -= length; - dumpBufferIfFull(); - return; - } - - System.arraycopy( b, offset, buffer, pos, avail ); - os.write( buffer ); - - offset += avail; - length -= avail; - - final int residual = length % buffer.length; - - os.write( b, offset, length - residual ); - System.arraycopy( b, offset + length - residual, buffer, 0, residual ); - pos = residual; - avail = buffer.length - residual; - } - - public void close() throws IOException { - if ( os == null ) return; - if ( pos != 0 ) os.write( buffer, 0, pos ); - if ( os != System.out ) os.close(); - os = null; - buffer = null; - } - } -// Local Variables: -// mode: jde -// tab-width: 4 -// End: - diff --git a/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java b/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java deleted file mode 100644 index 5fbd9b39..00000000 --- a/modules/src/main/java/org/archive/modules/deciderules/surt/ScopePlusOneDecideRule.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * This file is part of the Heritrix web crawler (crawler.archive.org). - * - * Licensed to the Internet Archive (IA) by one or more individual - * contributors. - * - * The IA licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.archive.modules.deciderules.surt; - -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.archive.modules.CrawlURI; -import org.archive.modules.deciderules.DecideResult; -import org.archive.net.UURI; -import org.archive.util.SurtPrefixSet; - -/** - * Rule allows one level of discovery beyond configured scope - * (e.g. Domain, plus the first otherwise out-of-scope link from an - * in-scope page, but not further hops from that first page) - * - * @author Shifra Raffel - * @version $Date$ $Revision$ - */ -public class ScopePlusOneDecideRule extends SurtPrefixedDecideRule { - - private static final long serialVersionUID = 3L; - - { - setUseDomain(true); - } - public boolean getUseDomain() { - return (Boolean)kp.get("useDomain"); - } - public void setUseDomain(boolean useDomain) { - kp.put("useDomain", useDomain); - } - - private static final Logger logger = - Logger.getLogger(ScopePlusOneDecideRule.class.getName()); - - /** - * Constructor. - * @param name - */ - public ScopePlusOneDecideRule() { - super(); - } - - /** - * Evaluate whether given object comes from a URI which is in scope - * - * @param object to evaluate - * @return true if URI is either in scope or its via is - */ - @Override - protected DecideResult innerDecide(CrawlURI uri) { - SurtPrefixSet set = getPrefixes(); - UURI u = uri.getUURI(); - // First, is the URI itself in scope? - boolean firstResult = isInScope(u, set); - if (logger.isLoggable(Level.FINE)) { - logger.fine("Tested scope of UURI itself '" + u + - " and result was " + firstResult); - } - if (firstResult == true) { - return DecideResult.ACCEPT; - } else { - // This object is not itself within scope, but - // see whether its via might be - UURI via = uri.getVia(); - if (via == null) { - // If there is no via and the URL doesn't match scope,reject it - return DecideResult.NONE; - } - // If the via is within scope, accept it - boolean result = isInScope (via, set); - if (logger.isLoggable(Level.FINE)) { - logger.fine("Tested via UURI '" + via + - " and result was " + result); - } - if (result) { - return DecideResult.ACCEPT; - } - } - return DecideResult.NONE; - } - - /** - * Synchronized get of prefix set to use. - * @param o Context object. - * - * @return SurtPrefixSet to use for check - * @see org.archive.modules.deciderules.surt.SurtPrefixedDecideRule#getPrefixes() - */ - protected synchronized SurtPrefixSet getPrefixes(/*StateProvider o*/) { - if (surtPrefixes == null) { - readPrefixes(); - } - return surtPrefixes; - } - - /** - * Patch the SURT prefix set so that it only includes the appropriate - * prefixes. - * @param o Context object. - * @see org.archive.modules.deciderules.surt.SurtPrefixedDecideRule#readPrefixes() - */ - @Override - protected void readPrefixes() { - buildSurtPrefixSet(); - // See whether Host or Domain was chosen - if (getUseDomain()) { - surtPrefixes.convertAllPrefixesToDomains(); - } else { - surtPrefixes.convertAllPrefixesToHosts(); - } - dumpSurtPrefixSet(); - } - - - //check that the URI is in scope - private boolean isInScope (Object o, SurtPrefixSet set) { - boolean iResult = false; - UURI u = (UURI)o; - if (u == null) { - return false; - } - String candidateSurt = u.getSurtForm(); - // also want to treat https as http - if (candidateSurt.startsWith("https:")) { - candidateSurt = "http:" + candidateSurt.substring(6); - } - if (set.containsPrefixOf(candidateSurt)){ - iResult = true; - } - return iResult; - } -}