- * The 'Referer' header contans the location the crawler came from, the page - * the current URI was discovered in. The 'Referer' usually is logged on the - * remote server and can be of assistance to webmasters trying to figure how - * a crawler got to a particular area on a site. - */ - { - setSendReferer(true); - } - public boolean getSendReferer() { - return (Boolean) kp.get("sendReferer"); - } - public void setSendReferer(boolean sendClose) { - kp.put("sendReferer",sendClose); - } - - /** - * Send 'Range' header when a limit ({@link #MAX_LENGTH_BYTES}) on - * document size. - *
- * Be polite to the HTTP servers and send the 'Range' header, stating that
- * you are only interested in the first n bytes. Only pertinent if
- * {@link #MAX_LENGTH_BYTES} > 0. Sending the 'Range' header results in a
- * '206 Partial Content' status response, which is better than just cutting
- * the response mid-download. On rare occasion, sending 'Range' will
- * generate '416 Request Range Not Satisfiable' response.
- */
- {
- setSendRange(false);
- }
- public boolean getSendRange() {
- return (Boolean) kp.get("sendRange");
- }
- public void setSendRange(boolean sendRange) {
- kp.put("sendRange",sendRange);
- }
-
- /**
- * Send 'If-Modified-Since' header, if previous 'Last-Modified' fetch
- * history information is available in URI history.
- */
- {
- setSendIfModifiedSince(true);
- }
- public boolean getSendIfModifiedSince() {
- return (Boolean) kp.get("sendIfModifiedSince");
- }
- public void setSendIfModifiedSince(boolean sendIfModifiedSince) {
- kp.put("sendIfModifiedSince",sendIfModifiedSince);
- }
-
- /**
- * Send 'If-None-Match' header, if previous 'Etag' fetch history information
- * is available in URI history.
- */
- {
- setSendIfNoneMatch(true);
- }
- public boolean getSendIfNoneMatch() {
- return (Boolean) kp.get("sendIfNoneMatch");
- }
- public void setSendIfNoneMatch(boolean sendIfNoneMatch) {
- kp.put("sendIfNoneMatch",sendIfNoneMatch);
- }
-
- public static final String REFERER = "Referer";
-
- public static final String RANGE = "Range";
-
- public static final String RANGE_PREFIX = "bytes=0-";
-
- public static final String HTTP_SCHEME = "http";
-
- public static final String HTTPS_SCHEME = "https";
-
-
- protected CookieStorage cookieStorage = new BdbCookieStorage();
- @Autowired(required=false)
- public void setCookieStorage(CookieStorage storage) {
- this.cookieStorage = storage;
- }
- public CookieStorage getCookieStorage() {
- return this.cookieStorage;
- }
-
- /**
- * Disable cookie handling.
- */
- {
- setIgnoreCookies(false);
- }
- public boolean getIgnoreCookies() {
- return (Boolean) kp.get("ignoreCookies");
- }
- public void setIgnoreCookies(boolean ignoreCookies) {
- kp.put("ignoreCookies",ignoreCookies);
- }
-
- /**
- * Local IP address or hostname to use when making connections (binding
- * sockets). When not specified, uses default local address(es).
- */
- public String getHttpBindAddress(){
- return (String) kp.get(HTTP_BIND_ADDRESS);
- }
- public void setHttpBindAddress(String address) {
- kp.put(HTTP_BIND_ADDRESS, address);
- }
- public static final String HTTP_BIND_ADDRESS = "httpBindAddress";
-
- /**
- * Used to store credentials.
- */
- {
- // initialize with empty store so declaration not required
- setCredentialStore(new CredentialStore());
- }
- public CredentialStore getCredentialStore() {
- return (CredentialStore) kp.get("credentialStore");
- }
- @Autowired(required=false)
- public void setCredentialStore(CredentialStore credentials) {
- kp.put("credentialStore",credentials);
- }
-
- /**
- * Used to do DNS lookups.
- */
- protected ServerCache serverCache;
- public ServerCache getServerCache() {
- return this.serverCache;
- }
- @Autowired
- public void setServerCache(ServerCache serverCache) {
- this.serverCache = serverCache;
- }
-
- static {
- Protocol.registerProtocol("http", new Protocol("http",
- new HeritrixProtocolSocketFactory(), 80));
- try {
- ProtocolSocketFactory psf = new HeritrixSSLProtocolSocketFactory();
- Protocol p = new Protocol("https", psf, 443);
- Protocol.registerProtocol("https", p);
- } catch (KeyManagementException e) {
- e.printStackTrace();
- } catch (KeyStoreException e) {
- e.printStackTrace();
- } catch (NoSuchAlgorithmException e) {
- e.printStackTrace();
- }
- }
-
-
- // static final String SERVER_CACHE_KEY = "heritrix.server.cache";
- static final String SSL_FACTORY_KEY = "heritrix.ssl.factory";
-
- /***************************************************************************
- * Socket factory that has the configurable trust manager installed.
- */
- private transient SSLSocketFactory sslfactory = null;
-
- /**
- * Constructor.
- */
- public FetchHTTP() {
- }
-
- protected void innerProcess(final CrawlURI curi)
- throws InterruptedException {
- // Note begin time
- curi.setFetchBeginTime(System.currentTimeMillis());
-
- // Get a reference to the HttpRecorder that is set into this ToeThread.
- Recorder rec = curi.getRecorder();
-
- // Shall we get a digest on the content downloaded?
- boolean digestContent = getDigestContent();
- String algorithm = null;
- if (digestContent) {
- algorithm = getDigestAlgorithm();
- rec.getRecordedInput().setDigest(algorithm);
- } else {
- // clear
- rec.getRecordedInput().setDigest((MessageDigest)null);
- }
-
- // Below we do two inner classes that add check of midfetch
- // filters just as we're about to receive the response body.
- String curiString = curi.getUURI().toString();
- HttpMethodBase method = null;
- if (curi.getFetchType() == HTTP_POST) {
- method = new HttpRecorderPostMethod(curiString, rec) {
- protected void readResponseBody(HttpState state,
- HttpConnection conn) throws IOException, HttpException {
- addResponseContent(this, curi);
- if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) {
- doAbort(curi, this, MIDFETCH_ABORT_LOG);
- } else {
- super.readResponseBody(state, conn);
- }
- }
- };
- curi.setFetchType(FetchType.HTTP_POST);
- } else {
- method = new HttpRecorderGetMethod(curiString, rec) {
- protected void readResponseBody(HttpState state,
- HttpConnection conn) throws IOException, HttpException {
- addResponseContent(this, curi);
- if (checkMidfetchAbort(curi, this.httpRecorderMethod, conn)) {
- doAbort(curi, this, MIDFETCH_ABORT_LOG);
- } else {
- super.readResponseBody(state, conn);
- }
- }
- };
- curi.setFetchType(FetchType.HTTP_GET);
- }
-
-
- HostConfiguration customConfigOrNull = configureMethod(curi, method);
-
- // Populate credentials. Set config so auth. is not automatic.
- boolean addedCredentials = populateCredentials(curi, method);
- if (http.getState().getProxyCredentials(new AuthScope(getProxyHost(), getProxyPort())) != null) {
- addedCredentials = true;
- }
-
- // set hardMax on bytes (if set by operator)
- long hardMax = getMaxLengthBytes();
- // set overall timeout (if set by operator)
- long timeoutMs = 1000 * getTimeoutSeconds();
- // Get max fetch rate (bytes/ms). It comes in in KB/sec
- long maxRateKBps = getMaxFetchKBSec();
- rec.getRecordedInput().setLimits(hardMax, timeoutMs, maxRateKBps);
-
- try {
- this.http.executeMethod(customConfigOrNull, method);
- } catch (RecorderTooMuchHeaderException ex) {
- // when too much header material, abort like other truncations
- doAbort(curi, method, HEADER_TRUNC);
- } catch (IOException e) {
- failedExecuteCleanup(method, curi, e);
- return;
- } catch (ArrayIndexOutOfBoundsException e) {
- // For weird windows-only ArrayIndex exceptions in native
- // code... see
- // http://forum.java.sun.com/thread.jsp?forum=11&thread=378356
- // treating as if it were an IOException
- failedExecuteCleanup(method, curi, e);
- return;
- }
-
- // set softMax on bytes to get (if implied by content-length)
- long softMax = method.getResponseContentLength();
-
- try {
- if (!method.isAborted()) {
- // Force read-to-end, so that any socket hangs occur here,
- // not in later modules.
- rec.getRecordedInput().readFullyOrUntil(softMax);
- }
- } catch (RecorderTimeoutException ex) {
- doAbort(curi, method, TIMER_TRUNC);
- } catch (RecorderLengthExceededException ex) {
- doAbort(curi, method, LENGTH_TRUNC);
- } catch (IOException e) {
- cleanup(curi, e, "readFully", S_CONNECT_LOST);
- return;
- } catch (ArrayIndexOutOfBoundsException e) {
- // For weird windows-only ArrayIndex exceptions from native code
- // see http://forum.java.sun.com/thread.jsp?forum=11&thread=378356
- // treating as if it were an IOException
- cleanup(curi, e, "readFully", S_CONNECT_LOST);
- return;
- } finally {
- // ensure recording has stopped
- rec.closeRecorders();
- if (!method.isAborted()) {
- method.releaseConnection();
- }
- // Note completion time
- curi.setFetchCompletedTime(System.currentTimeMillis());
- // Set the response charset into the HttpRecord if available.
- setCharacterEncoding(curi, rec, method);
- setSizes(curi, rec);
- setOtherCodings(curi, rec, method);
- }
-
- if (digestContent) {
- curi.setContentDigest(algorithm,
- rec.getRecordedInput().getDigestValue());
- }
- if (logger.isLoggable(Level.FINE)) {
- logger.fine(((curi.getFetchType() == HTTP_POST) ? "POST" : "GET")
- + " " + curi.getUURI().toString() + " "
- + method.getStatusCode() + " "
- + rec.getRecordedInput().getSize() + " "
- + curi.getContentType());
- }
-
- if (isSuccess(curi) && addedCredentials) {
- // Promote the credentials from the CrawlURI to the CrawlServer
- // so they are available for all subsequent CrawlURIs on this
- // server.
- promoteCredentials(curi);
- if (logger.isLoggable(Level.FINE)) {
- // Print out the cookie. Might help with the debugging.
- Header setCookie = method.getResponseHeader("set-cookie");
- if (setCookie != null) {
- logger.fine(setCookie.toString().trim());
- }
- }
- } else if (method.getStatusCode() == HttpStatus.SC_UNAUTHORIZED) {
- // 401 is not 'success'.
- handle401(method, curi);
- }
-
- if (rec.getRecordedInput().isOpen()) {
- logger.severe(curi.toString() + " RIS still open. Should have"
- + " been closed by method release: "
- + Thread.currentThread().getName());
- try {
- rec.getRecordedInput().close();
- } catch (IOException e) {
- logger.log(Level.SEVERE, "second-chance RIS close failed", e);
- }
- }
- }
-
- /**
- * Update CrawlURI internal sizes based on current transaction (and
- * in the case of 304s, history)
- *
- * @param curi CrawlURI
- * @param rec HttpRecorder
- */
- protected void setSizes(CrawlURI curi, Recorder rec) {
- // set reporting size
- curi.setContentSize(rec.getRecordedInput().getSize());
- // special handling for 304-not modified
- if (curi.getFetchStatus() == HttpStatus.SC_NOT_MODIFIED
- && curi.containsDataKey(A_FETCH_HISTORY)) {
- Mapcuri with response status and
- * content type.
- *
- * @param curi
- * CrawlURI to populate.
- * @param method
- * Method to get response status and headers from.
- */
- protected void addResponseContent(HttpMethod method, CrawlURI curi) {
- curi.setFetchStatus(method.getStatusCode());
- Header ct = method.getResponseHeader("content-type");
- curi.setContentType((ct == null) ? null : ct.getValue());
-
- for (Header h: method.getResponseHeaders()) {
- curi.putHttpResponseHeader(h.getName(), h.getValue());
- }
- }
-
- /**
- * Set the character encoding based on the result headers or default.
- *
- * The HttpClient returns its own default encoding ("ISO-8859-1") if one
- * isn't specified in the Content-Type response header. We give the user the
- * option of overriding this, so we need to detect the case where the
- * default is returned.
- *
- * Now, it may well be the case that the default returned by HttpClient and
- * the default defined by the user are the same.
- *
- * TODO:FIXME?: This method does not do the "detect the case where the
- * [HttpClient] default is returned" mentioned above! Why not?
- *
- * @param rec
- * Recorder for this request.
- * @param method
- * Method used for the request.
- */
- private void setCharacterEncoding(CrawlURI curi, final Recorder rec,
- final HttpMethod method) {
- String encoding = ((HttpMethodBase) method).getResponseCharSet();
- try {
- rec.setCharset(Charset.forName(encoding));
- } catch (IllegalArgumentException e) {
- curi.getAnnotations().add("unsatisfiableCharsetInHeader:"+StringUtils.stripToEmpty(encoding));
- rec.setCharset(getDefaultCharset());
- }
- }
-
- /**
- * Set the transfer, content encodings based on headers (if necessary).
- *
- * @param rec
- * Recorder for this request.
- * @param method
- * Method used for the request.
- */
- private void setOtherCodings(CrawlURI uri, final Recorder rec,
- final HttpMethod method) {
- Header transferCodingHeader = ((HttpMethodBase) method).getResponseHeader("Transfer-Encoding");
- if (transferCodingHeader !=null) {
- String te = transferCodingHeader.getValue().trim();
- if(te.equalsIgnoreCase("chunked")) {
- rec.setInputIsChunked(true);
- } else {
- logger.log(Level.WARNING,"Unknown transfer-encoding '"+te+"' for "+uri.getURI());
- }
- }
- Header contentEncodingHeader = ((HttpMethodBase) method).getResponseHeader("Content-Encoding");
- if (contentEncodingHeader!=null) {
- String ce = contentEncodingHeader.getValue().trim();
- try {
- rec.setContentEncoding(ce);
- } catch (IllegalArgumentException e) {
- uri.getAnnotations().add("unsatisfiableContentEncoding:"+StringUtils.stripToEmpty(ce));
- }
- }
- }
-
- /**
- * Cleanup after a failed method execute.
- *
- * @param curi
- * CrawlURI we failed on.
- * @param method
- * Method we failed on.
- * @param exception
- * Exception we failed with.
- */
- private void failedExecuteCleanup(final HttpMethod method,
- final CrawlURI curi, final Exception exception) {
- cleanup(curi, exception, "executeMethod", (method.isRequestSent() ? S_CONNECT_LOST : S_CONNECT_FAILED));
- method.releaseConnection();
- }
-
- /**
- * Cleanup after a failed method execute.
- *
- * @param curi
- * CrawlURI we failed on.
- * @param exception
- * Exception we failed with.
- * @param message
- * Message to log with failure. FIXME: Seems ignored
- * @param status
- * Status to set on the fetch.
- */
- private void cleanup(final CrawlURI curi, final Exception exception,
- final String message, final int status) {
- // message ignored!
- curi.getNonFatalFailures().add(exception);
- curi.setFetchStatus(status);
- curi.getRecorder().close();
- }
-
- @Override
- public ProcessResult process(CrawlURI uri) throws InterruptedException {
- if (uri.getFetchStatus() < 0) {
- // already marked as errored, this pass through
- // skip to end
- return ProcessResult.FINISH;
- } else {
- return super.process(uri);
- }
- }
-
- /**
- * Can this processor fetch the given CrawlURI. May set a fetch status
- * if this processor would usually handle the CrawlURI, but cannot in
- * this instance.
- *
- * @param curi
- * @return True if processor can fetch.
- */
- @Override
- protected boolean shouldProcess(CrawlURI curi) {
- String scheme = curi.getUURI().getScheme();
- if (!(scheme.equals("http") || scheme.equals("https"))) {
- // handles only plain http and https
- return false;
- }
-
- CrawlHost host = serverCache.getHostFor(curi.getUURI());
- if (host.getIP() == null && host.hasBeenLookedUp()) {
- curi.setFetchStatus(S_DOMAIN_PREREQUISITE_FAILURE);
- return false;
- }
-
- return true;
- }
-
- /**
- * Configure the HttpMethod setting options and headers.
- *
- * @param curi
- * CrawlURI from which we pull configuration.
- * @param method
- * The Method to configure.
- */
- protected HostConfiguration configureMethod(CrawlURI curi,
- HttpMethod method) {
- // Don't try to handle 401s internally.
- method.setDoAuthentication(false);
-
- // Don't auto-follow redirects
- method.setFollowRedirects(false);
-
- // // set soTimeout
- // method.getParams().setSoTimeout(
- // ((Integer) getUncheckedAttribute(curi, ATTR_SOTIMEOUT_MS))
- // .intValue());
-
- // Set cookie policy.
- boolean ignoreCookies = getIgnoreCookies();
- method.getParams().setCookiePolicy(
- ignoreCookies ? CookiePolicy.IGNORE_COOKIES
- : CookiePolicy.BROWSER_COMPATIBILITY);
-
- method.getParams().setVersion(getUseHTTP11()
- ? HttpVersion.HTTP_1_1
- : HttpVersion.HTTP_1_0);
-
- UserAgentProvider uap = getUserAgentProvider();
- String from = uap.getFrom();
- String userAgent = curi.getUserAgent();
- if (userAgent == null) {
- userAgent = uap.getUserAgent();
- }
-
- method.setRequestHeader("User-Agent", userAgent);
- if(StringUtils.isNotBlank(from)) {
- method.setRequestHeader("From", from);
- }
-
- // Set retry handler.
- method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
- new HeritrixHttpMethodRetryHandler());
-
- final long maxLength = getMaxLengthBytes();
- if (maxLength > 0 && getSendRange()) {
- method.addRequestHeader(RANGE, RANGE_PREFIX.concat(Long
- .toString(maxLength - 1)));
- }
-
- if (getSendConnectionClose()) {
- method.addRequestHeader(HEADER_SEND_CONNECTION_CLOSE);
- }
-
- if (getSendReferer() && !LinkContext.PREREQ_MISC.equals(curi.getViaContext())) {
- // RFC2616 says no referer header if referer is https and the url
- // is not
- String via = flattenVia(curi);
- if (via != null
- && via.length() > 0
- && !(via.startsWith(HTTPS_SCHEME) && curi.getUURI()
- .getScheme().equals(HTTP_SCHEME))) {
- method.setRequestHeader(REFERER, via);
- }
- }
-
- if (!curi.isPrerequisite()) {
- setConditionalGetHeader(curi, method, getSendIfModifiedSince(),
- A_LAST_MODIFIED_HEADER, "If-Modified-Since");
- setConditionalGetHeader(curi, method, getSendIfNoneMatch(),
- A_ETAG_HEADER, "If-None-Match");
- }
-
- // TODO: What happens if below method adds a header already
- // added above: e.g. Connection, Range, or Referer?
- setAcceptHeaders(curi, method);
-
- HostConfiguration config =
- new HostConfiguration(http.getHostConfiguration());
- configureProxy(curi, config);
- configureBindAddress(curi, config);
- return config;
- }
-
- /**
- * Set the given conditional-GET header, if the setting is enabled and
- * a suitable value is available in the URI history.
- * @param curi source CrawlURI
- * @param method HTTP operation pending
- * @param setting true/false enablement setting name to consult
- * @param sourceHeader header to consult in URI history
- * @param targetHeader header to set if possible
- */
- protected void setConditionalGetHeader(CrawlURI curi, HttpMethod method,
- boolean conditional, String sourceHeader, String targetHeader) {
- if (conditional) {
- try {
- HashMapmethod.
- *
- * Do credential handling. Credentials are in two places. 1. Credentials
- * that succeeded are added to the CrawlServer (Or rather, avatars for
- * credentials are whats added because its not safe to keep around
- * references to credentials). 2. Credentials to be tried are in the curi.
- * Returns true if found credentials to be tried.
- *
- * @param curi
- * Current CrawlURI.
- * @param method
- * The method to add to.
- * @return True if prepopulated method with credentials AND
- * the credentials came from the curi, not from the
- * CrawlServer. The former is special in that if the
- * curi credentials
- * succeed, then the caller needs to promote them from the CrawlURI to the
- * CrawlServer so they are available for all subsequent CrawlURIs on this
- * server.
- */
- private boolean populateCredentials(CrawlURI curi, HttpMethod method) {
- // First look at the server avatars. Add any that are to be volunteered
- // on every request (e.g. RFC2617 credentials). Every time creds will
- // return true when we call 'isEveryTime().
- String serverKey;
- try {
- serverKey = CrawlServer.getServerKey(curi.getUURI());
- } catch (URIException e) {
- return false;
- }
- CrawlServer server = serverCache.getServerFor(serverKey);
- if (server.hasCredentials()) {
- for (Credential cred : server.getCredentials()) {
- if (cred.isEveryTime()) {
- CommonsHttpCredentialUtil.populate(curi, this.http, method, cred, server.getHttpAuthChallenges());
- }
- }
- }
-
- boolean result = false;
-
- // Now look in the curi. The Curi will have credentials loaded either
- // by the handle401 method if its a rfc2617 or it'll have been set into
- // the curi by the preconditionenforcer as this login uri came through.
- for (Credential c: curi.getCredentials()) {
- if (CommonsHttpCredentialUtil.populate(curi, this.http, method, c, curi.getHttpAuthChallenges())) {
- result = true;
- }
- }
-
- return result;
- }
-
- /**
- * Promote successful credential to the server.
- *
- * @param curi
- * CrawlURI whose credentials we are to promote.
- */
- private void promoteCredentials(final CrawlURI curi) {
- Set