From 0dbc773fc8d3bdf21fa47f8aa0a74fe661766de8 Mon Sep 17 00:00:00 2001
From: nlevitt <nlevitt@daa5b2f2-a927-0410-8b2d-f5f262fa301a>
Date: Sat, 15 May 2010 02:13:19 +0000
Subject: [PATCH] Followup on HER-1728 - treat flashvars strings more like url
 query strings (key=value&[key=value]*) * ExtractorHTML.java    
 considerQueryStringValues(): split pairs on "&" instead of ";" and    
 url-decode values

---
 .../modules/extractor/ExtractorHTML.java       | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
index 440a7103..0584aa36 100644
--- a/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
+++ b/modules/src/main/java/org/archive/modules/extractor/ExtractorHTML.java
@@ -20,6 +20,8 @@
 package org.archive.modules.extractor;
 
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.logging.Level;
@@ -509,7 +511,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
                     && "flashvars".equalsIgnoreCase(nameVal.toString())) {
                 // special handling for <PARAM NAME='flashvars" VALUE="">
                 String queryStringLike = valueVal.toString();
-                // treat value as query-string-like "key=value[;key=value]*" pairings
+                // treat value as query-string-like "key=value[&key=value]*" pairings
                 considerQueryStringValues(curi, queryStringLike, valueContext,Hop.SPECULATIVE);
             } else {
                 // regular VALUE handling
@@ -521,7 +523,7 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
     }
 
     /**
-     * Consider a query-string-like collections of key=value[;key=value]
+     * Consider a query-string-like collections of key=value[&key=value]
      * pairs for URI-like strings in the values. Where URI-like strings are
      * found, add as discovered outlink. 
      * 
@@ -531,14 +533,18 @@ public class ExtractorHTML extends ContentExtractor implements InitializingBean
      */
     protected void considerQueryStringValues(CrawlURI curi,
             CharSequence queryString, CharSequence valueContext, Hop hop) {
-        for(String pairString : queryString.toString().split(";")) {
-            String[] keyVal = pairString.split("=");
-            if(keyVal.length==2) {
-                considerIfLikelyUri(curi,keyVal[1],valueContext, hop);
+        for (String pairString : queryString.toString().split("&")) {
+            String[] encodedKeyVal = pairString.split("=");
+            if (encodedKeyVal.length == 2) try {
+                String value = URLDecoder.decode(encodedKeyVal[1], "UTF-8");
+                considerIfLikelyUri(curi, value, valueContext, hop);
+            } catch (UnsupportedEncodingException e) {
+                throw new AssertionError("all jvms must support UTF-8, and yet somehow this happened: " + e);
             }
         }
     }
 
+
     /**
      * Consider whether a given string is URI-like. If so, add as discovered 
      * outlink.