Merge branch 'master' into trough-dedup

* master:
  replace System.err.println with logger.info
  Revert "Upgrade httpclient to 4.5.7 and handle cookies more compliantly"
  Removing outdated test.
  Disable questionalbe test.
  Avoid deprecated flag.
  Supply an iterator, for #245
  Updated POM to use latest version.
  Update README.md
  Handle missing closing paren in srcset descriptor
  Teach jericho extractor srcset
  Don't run srcset test against jericho, it doesn't handle it
  Handle commas more compliantly when parsing srcset
  Ensure we start parsing full lines, for #239.

README.md

@@ -32,7 +32,7 @@ you or adapt their server behavior accordingly.
 
 ## 3. Getting Started
 
-See the User Manual, available from <https://github.com/internetarchive/heritrix3/wiki
+See the User Manual, available from <https://github.com/internetarchive/heritrix3/wiki>
 
 
 ## 4. Developer Documentation

engine/src/main/java/org/archive/crawler/framework/CrawlJob.java

@@ -172,6 +172,12 @@ public class CrawlJob implements Comparable<CrawlJob>, ApplicationListener<Appli
             BufferedReader jobLogReader = new BufferedReader(
                     new InputStreamReader(jobLogIn));
             String line;
+            // If we sliced into the file, make sure we skip to the next line:
+            // (See https://github.com/internetarchive/heritrix3/issues/239)
+            if (startPosition != 0) {
+                line = jobLogReader.readLine();
+            }
+            // Parse lines looking for launch details:
             while ((line = jobLogReader.readLine()) != null) {
                 Matcher m = launchLine.matcher(line);
                 if (m.matches()) {

engine/src/main/java/org/archive/crawler/frontier/WorkQueueFrontier.java

@@ -442,8 +442,8 @@ implements Closeable,
         synchronized(wq) {
             wq.noteDeactivated();
             inProcessQueues.remove(wq);
-            if(wq.getCount()==0) {
-                System.err.println("deactivate empty queue?");
+            if (wq.getCount() == 0) {
+                logger.info("deactivate empty queue? " + wq.getClassKey());
             }
 
             synchronized (getInactiveQueuesByPrecedence()) {

modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java

@@ -682,13 +682,15 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
 
             logger.fine("Found srcset listing: " + value.toString());
 
-            String[] links = value.toString().split(",");
-            for (int i=0; i < links.length; i++){
-                String link = links[i].trim().split(" +")[0];
+            Matcher matcher = TextUtils.getMatcher("[\\s,]*(\\S*[^,\\s])(?:\\s(?:[^,(]+|\\([^)]*(?:\\)|$))*)?", value);
+            while (matcher.lookingAt()) {
+                String link = matcher.group(1);
+                matcher.region(matcher.end(), matcher.regionEnd());
                 logger.finer("Found " + link + " adding to outlinks.");
                 addLinkFromString(curi, link, context, hop);
                 numberOfLinksExtracted.incrementAndGet();
             }
+            TextUtils.recycleMatcher(matcher);
         } else {
             addLinkFromString(curi,
                 (value instanceof String)?

modules/src/main/java/org/archive/modules/extractor/JerichoExtractorHTML.java

@@ -196,6 +196,13 @@ public class JerichoExtractorHTML extends ExtractorHTML {
 
             processEmbed(curi, attrValue, context, hopType);
         }
+        // SRCSET
+        if (((attr = attributes.get("srcset")) != null) &&
+                ((attrValue = attr.getValue()) != null)) {
+            codebase = StringEscapeUtils.unescapeHtml(attrValue);
+            CharSequence context = elementContext(elementName, attr.getKey());
+            processEmbed(curi, codebase, context);
+        }
         // CODEBASE
         if (((attr = attributes.get("codebase")) != null) &&
                  ((attrValue = attr.getValue()) != null)) {

modules/src/test/java/org/archive/modules/extractor/ExtractorHTMLTest.java

@@ -512,7 +512,7 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
 
         CharSequence cs = "<img width=\"800\" height=\"1200\" src=\"/images/foo.jpg\" "
                 + "class=\"attachment-full size-full\" alt=\"\" "
-                + "srcset=\"/images/foo1.jpg 800w, /images/foo2.jpg 480w, /images/foo3.jpg 96w\" "
+                + "srcset=\"a,b,c,,, /images/foo1.jpg 800w,data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7 700w, /images/foo2.jpg 480w(data:,foo, ,), /images/foo3.jpg 96w(x\" "
                 + "sizes=\"(max-width: 800px) 100vw, 800px\">";
 
         getExtractor().extract(curi, cs);
@@ -521,6 +521,8 @@ public class ExtractorHTMLTest extends StringExtractorTestBase {
         Arrays.sort(links);
 
         String[] dest = {
+                "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7",
+                "http://www.example.com/a,b,c",
                 "http://www.example.com/images/foo.jpg",
                 "http://www.example.com/images/foo1.jpg",
                 "http://www.example.com/images/foo2.jpg",

modules/src/test/java/org/archive/modules/extractor/JerichoExtractorHTMLTest.java

@@ -151,7 +151,7 @@ public class JerichoExtractorHTMLTest extends ExtractorHTMLTest {
     @Override
     public void testConditionalComment1() throws URIException {
     }
-    
+
     /*
      * Override of ExtractorHTMLTest method because the test fails with
      * JerichoExtractorHTML