From 252cf63863a201625b7e822349062df8ccd3eb7d Mon Sep 17 00:00:00 2001 From: gojomo Date: Tue, 12 May 2009 22:09:22 +0000 Subject: [PATCH] Fix for [HER-1524] speculativeFixup in ExtractorJS should maintain URL scheme * ExtractorHTMLTest.java test for maintaining 'https' when speculative, leading-domain URI is on same host * ExtractorJS.java (speculativeFixup) maintain origin URI scheme if apparent target host is same Fix for [HER-1526] SCRIPT writing script TYPE common trigger of bogus links (eg. 'text/javascript') Fix for [HER-1611] more javascript false-positives from Google Analytics insert * ExtractorJS.java port of Steve's work in h2: blacklist for URIs, extracted strings to handle common mis-extractions * ExtractorHTMLTest.java port of test for 'text/javascript' false extract --- .project | 34 ++--- .../modules/extractor/ExtractorJS.java | 130 +++++++++++++++--- .../modules/extractor/ExtractorHTMLTest.java | 61 ++++++++ 3 files changed, 189 insertions(+), 36 deletions(-) diff --git a/.project b/.project index a3c7cfba..24c1cd6b 100644 --- a/.project +++ b/.project @@ -1,17 +1,17 @@ - - - crawler - - - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.core.javanature - - + + + heritrix3 + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java index 9fd1d57c..544f8779 100644 --- a/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java +++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorJS.java @@ -1,35 +1,34 @@ -/* Copyright (C) 2003 Internet Archive. +/* + * This file is part of the Heritrix web crawler (crawler.archive.org). * - * This file is part of the Heritrix web crawler (crawler.archive.org). + * Licensed to the Internet Archive (IA) by one or more individual + * contributors. * - * Heritrix is free software; you can redistribute it and/or modify - * it under the terms of the GNU Lesser Public License as published by - * the Free Software Foundation; either version 2.1 of the License, or - * any later version. + * The IA licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * - * Heritrix is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser Public License for more details. + * http://www.apache.org/licenses/LICENSE-2.0 * - * You should have received a copy of the GNU Lesser Public License - * along with Heritrix; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Created on Nov 17, 2003 - * - * To change the template for this generated file go to - * Window>Preferences>Java>Code Generation>Code and Comments + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + package org.archive.modules.extractor; import java.io.IOException; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; +import org.apache.commons.codec.DecoderException; import org.apache.commons.httpclient.URIException; import org.archive.io.ReplayCharSequence; import org.archive.modules.ProcessorURI; +import org.archive.net.LaxURLCodec; import org.archive.net.UURI; import org.archive.util.DevUtils; import org.archive.util.TextUtils; @@ -73,6 +72,22 @@ public class ExtractorJS extends ContentExtractor { protected long numberOfCURIsHandled = 0; protected static long numberOfLinksExtracted = 0; + // strings that STRING_URI_DETECTOR picks up as URIs, + // which are known to be problematic, and NOT to be + // added to outLinks + protected final static String[] STRING_URI_DETECTOR_EXCEPTIONS = { + "text/javascript" + }; + + // URIs known to produce false-positives with the current JS extractor. + // e.g. currently (2.0.3) the JS extractor produces 13 false-positive + // URIs from http://www.google-analytics.com/urchin.js and only 2 + // good URIs, which are merely one pixel images. + // TODO: remove this blacklist when JS extractor is improved + protected final static String[] EXTRACTOR_URI_EXCEPTIONS = { + "http://www.google-analytics.com/urchin.js" + }; + /** * @param name */ @@ -81,6 +96,14 @@ public class ExtractorJS extends ContentExtractor { protected boolean shouldExtract(ProcessorURI uri) { + + // special-cases, for when we know our current JS extractor does poorly. + // TODO: remove this test when JS extractor is improved + for (String s: EXTRACTOR_URI_EXCEPTIONS) { + if (uri.toString().equals(s)) + return false; + } + String contentType = uri.getContentType(); if ((contentType == null)) { return false; @@ -163,7 +186,12 @@ public class ExtractorJS extends ContentExtractor { TextUtils.getMatcher(STRING_URI_DETECTOR, subsequence); if(uri.matches()) { String string = uri.group(); - string = TextUtils.replaceAll(ESCAPED_AMP, string, AMP); + // protect against adding outlinks for known problematic matches + if (isUriMatchException(string,cs)) { + TextUtils.recycleMatcher(uri); + continue; + } + string = speculativeFixup(string, curi); foundLinks++; try { int max = ext.getExtractorParameters().getMaxOutlinks(); @@ -186,6 +214,70 @@ public class ExtractorJS extends ContentExtractor { TextUtils.recycleMatcher(strings); return foundLinks; } + + /** + * checks to see if URI match is a special case + * @param string matched by STRING_URI_DETECTOR + * @param cs + * @return true if string is one of STRING_URI_EXCEPTIONS + */ + private static boolean isUriMatchException(String string,CharSequence cs) { + for (String s : STRING_URI_DETECTOR_EXCEPTIONS) { + if (s.equals(string)) + return true; + } + return false; + } + + + /** + * Perform additional fixup of likely-URI Strings + * + * @param string detected candidate String + * @return String changed/decoded to increase liklihood it is a + * meaningful non-404 URI + */ + public static String speculativeFixup(String string, ProcessorURI puri) { + String retVal = string; + + // unescape ampersands + retVal = TextUtils.replaceAll(ESCAPED_AMP, retVal, AMP); + + // uri-decode if begins with encoded 'http(s)?%3A' + Matcher m = TextUtils.getMatcher("(?i)^https?%3A.*",retVal); + if(m.matches()) { + try { + retVal = LaxURLCodec.DEFAULT.decode(retVal); + } catch (DecoderException e) { + LOGGER.log(Level.INFO,"unable to decode",e); + } + } + TextUtils.recycleMatcher(m); + + // TODO: more URI-decoding if there are %-encoded parts? + + // detect scheme-less intended-absolute-URI + // intent: "opens with what looks like a dotted-domain, and + // last segment is a top-level-domain (eg "com", "org", etc)" + m = TextUtils.getMatcher( + "^[^\\./:\\s%]+\\.[^/:\\s%]+\\.([^\\./:\\s%]+)(/.*|)$", + retVal); + if(m.matches()) { + String schemePlus = "http://"; + // if on exact same host preserve scheme (eg https) + try { + if (retVal.startsWith(puri.getUURI().getHost())) { + schemePlus = puri.getUURI().getScheme() + "://"; + } + } catch (URIException e) { + // error retrieving source host - ignore it + } + retVal = schemePlus + retVal; + } + TextUtils.recycleMatcher(m); + + return retVal; + } /* (non-Javadoc) * @see org.archive.crawler.framework.Processor#report() diff --git a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java index 8dc5f9ea..580c2ff3 100644 --- a/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java +++ b/modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java @@ -23,6 +23,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.Predicate; import org.apache.commons.httpclient.URIException; import org.archive.modules.DefaultProcessorURI; import org.archive.net.UURI; @@ -201,4 +203,63 @@ public class ExtractorHTMLTest extends StringExtractorTestBase { Link[] links = puri.getOutLinks().toArray(new Link[0]); assertTrue("link extracted despite meta robots",links.length==0); } + + /** + * Test if scheme is maintained by speculative hops onto exact + * same host + * + * [HER-1524] speculativeFixup in ExtractorJS should maintain URL scheme + */ + public void testSpeculativeLinkExtraction() throws URIException { + DefaultProcessorURI curi = new DefaultProcessorURI(UURIFactory + .getInstance("https://www.example.com"), null); + CharSequence cs = + ""; + ExtractorHTML extractor = (ExtractorHTML)makeExtractor(); + extractor.extract(curi, cs); + + assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { + public boolean evaluate(Object object) { + System.err.println("comparing: " + + ((Link) object).getDestination().toString() + + " and https://www.anotherexample.com/"); + return ((Link) object).getDestination().toString().equals( + "http://www.anotherexample.com/"); + } + })); + assertTrue(CollectionUtils.exists(curi.getOutLinks(), new Predicate() { + public boolean evaluate(Object object) { + return ((Link) object).getDestination().toString().equals( + "https://www.example.com/index.html"); + } + })); + } + + /** + * test to see if embedded "; + ExtractorHTML extractor = (ExtractorHTML)makeExtractor(); + extractor.extract(curi, cs); + assertTrue("outlinks should be empty",curi.getOutLinks().isEmpty()); + + } }