diff --git a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java index cab71424..9bdec485 100644 --- a/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java +++ b/contrib/src/main/java/org/archive/modules/extractor/ExtractorYoutubeFormatStream.java @@ -28,180 +28,180 @@ import org.json.JSONObject; * */ public class ExtractorYoutubeFormatStream extends Extractor { - private static Logger logger = + private static Logger logger = Logger.getLogger(ExtractorYoutubeFormatStream.class.getName()); { - setExtractLimit(1); + setExtractLimit(1); } public Integer getExtractLimit(){ - return (Integer) kp.get("extractLimit"); + return (Integer) kp.get("extractLimit"); } /** * Maximum number of video urls to extract. A value of 0 means extract all * discovered video urls. Default is 1. */ public void setExtractLimit(Integer extractLimit){ - kp.put("extractLimit", extractLimit); + kp.put("extractLimit", extractLimit); } { - setItagPriority(new ArrayList()); + setItagPriority(new ArrayList()); } @SuppressWarnings("unchecked") public List getItagPriority() { - return (List) kp.get("itagPriority"); + return (List) kp.get("itagPriority"); } - - /** - * Itag priority list. Youtube itag parameter specifies the video and audio - * format and quality. The default is an empty list, which tells the - * extractor to extract up to extractLimit video urls. When the - * list is not empty, only video urls with itag values in the list are - * extracted. - * - * @see http://en.wikipedia.org/wiki/YouTube - */ + + /** + * Itag priority list. Youtube itag parameter specifies the video and audio + * format and quality. The default is an empty list, which tells the + * extractor to extract up to extractLimit video urls. When the + * list is not empty, only video urls with itag values in the list are + * extracted. + * + * @see http://en.wikipedia.org/wiki/YouTube + */ public void setItagPriority(List itagPriority) { - kp.put("itagPriority", itagPriority); + kp.put("itagPriority", itagPriority); } - @Override + @Override protected boolean shouldProcess(CrawlURI uri) { - return uri.getContentLength() > 0 - && uri.getFetchStatus() == 200 - && TextUtils.matches("^https?://([^.]+\\.)?youtube\\.com/watch.*$", - uri.getUURI().toCustomString()); + return uri.getContentLength() > 0 + && uri.getFetchStatus() == 200 + && TextUtils.matches("^https?://([^.]+\\.)?youtube\\.com/watch.*$", + uri.getUURI().toCustomString()); } - @Override - protected void extract(CrawlURI uri) { - ReplayCharSequence cs; - try { - cs = uri.getRecorder().getContentReplayCharSequence(); - } catch (IOException e) { - uri.getNonFatalFailures().add(e); - logger.log(Level.WARNING, "Failed get of replay char sequence in " - + Thread.currentThread().getName(), e); - return; - } + @Override + protected void extract(CrawlURI uri) { + ReplayCharSequence cs; + try { + cs = uri.getRecorder().getContentReplayCharSequence(); + } catch (IOException e) { + uri.getNonFatalFailures().add(e); + logger.log(Level.WARNING, "Failed get of replay char sequence in " + + Thread.currentThread().getName(), e); + return; + } - Matcher matcher = TextUtils.getMatcher( - "(?is)ytplayer\\.config = (\\{.*?\\})(;||$)", cs); - if (matcher.find()) { - String jsonStr = matcher.group(1); + Matcher matcher = TextUtils.getMatcher( + "(?is)ytplayer\\.config = (\\{.*?\\})(;||$)", cs); + if (matcher.find()) { + String jsonStr = matcher.group(1); - // logger.fine("Just Extracted: "+jsonStr); - try { - JSONObject json = new JSONObject(jsonStr); - if (json.has("args")) { - JSONObject args = json.getJSONObject("args"); - if (args.has("url_encoded_fmt_stream_map")) { - String streamMap = args.getString("url_encoded_fmt_stream_map"); + // logger.fine("Just Extracted: "+jsonStr); + try { + JSONObject json = new JSONObject(jsonStr); + if (json.has("args")) { + JSONObject args = json.getJSONObject("args"); + if (args.has("url_encoded_fmt_stream_map")) { + String streamMap = args.getString("url_encoded_fmt_stream_map"); - // logger.info("Just Extracted: "+stream_map); - LinkedHashMap parsedVideoMap = parseStreamMap(streamMap); - addPreferredOutlinks(uri, parsedVideoMap); - } - } - } catch (JSONException e) { - logger.log(Level.WARNING, - "Error parsing JSON object - Skipping: " + jsonStr, e); - } - } - TextUtils.recycleMatcher(matcher); - } + // logger.info("Just Extracted: "+stream_map); + LinkedHashMap parsedVideoMap = parseStreamMap(streamMap); + addPreferredOutlinks(uri, parsedVideoMap); + } + } + } catch (JSONException e) { + logger.log(Level.WARNING, + "Error parsing JSON object - Skipping: " + jsonStr, e); + } + } + TextUtils.recycleMatcher(matcher); + } - // 34 and 35 are most common medium quality flvs, others are in arbitrary order - private static final List DEFAULT_ITAG_PRIORITY = Arrays.asList( - "35", "34", "5", "6", "13", "17", "18", "22", "36", "37", "38", - "43", "44", "45", "46", "82", "83", "84", "85", "100", "101", - "102", "120"); - private static final Set KNOWN_ITAGS = new HashSet(DEFAULT_ITAG_PRIORITY); - - // Add videos as outlinks by priority list - private void addPreferredOutlinks(CrawlURI uri, - LinkedHashMap parsedVideoMap) { - List itagPriority; - if (getItagPriority() != null && !getItagPriority().isEmpty()) { - itagPriority = getItagPriority(); - } else { - itagPriority = DEFAULT_ITAG_PRIORITY; - } - - int extractionCount = 0; - for (String itag : itagPriority) { - if (parsedVideoMap.containsKey(itag) - && (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) { - logger.fine("adding video: " + parsedVideoMap.get(itag)); - addOutlink(uri, parsedVideoMap.get(itag), - org.archive.modules.extractor.LinkContext.EMBED_MISC, - org.archive.modules.extractor.Hop.EMBED); - extractionCount++; - } - } + // 34 and 35 are most common medium quality flvs, others are in arbitrary order + private static final List DEFAULT_ITAG_PRIORITY = Arrays.asList( + "35", "34", "5", "6", "13", "17", "18", "22", "36", "37", "38", + "43", "44", "45", "46", "82", "83", "84", "85", "100", "101", + "102", "120"); + private static final Set KNOWN_ITAGS = new HashSet(DEFAULT_ITAG_PRIORITY); - // if itagPriority not specified, make sure we consider all discovered - // video urls - if (getItagPriority() == null || getItagPriority().isEmpty()) { - Iterator itagKeyIter = parsedVideoMap.keySet().iterator(); - while (itagKeyIter.hasNext() && (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) { - String itag = itagKeyIter.next(); - if (!KNOWN_ITAGS.contains(itag)) { - logger.warning("adding video (with unknown itag " + itag - + "): " + parsedVideoMap.get(itag)); - addOutlink(uri, parsedVideoMap.get(itag), - org.archive.modules.extractor.LinkContext.EMBED_MISC, - org.archive.modules.extractor.Hop.EMBED); - extractionCount++; - } - } - } - } - - private LinkedHashMap parseStreamMap(String streamMap) { - String[] rawVideoList = streamMap.split(","); - LinkedHashMap parsedVideoMap = new LinkedHashMap(); + // Add videos as outlinks by priority list + private void addPreferredOutlinks(CrawlURI uri, + LinkedHashMap parsedVideoMap) { + List itagPriority; + if (getItagPriority() != null && !getItagPriority().isEmpty()) { + itagPriority = getItagPriority(); + } else { + itagPriority = DEFAULT_ITAG_PRIORITY; + } - // Parse Video Map into itag,url pair - for (int i = 0; i < rawVideoList.length; i++) { - String[] videoParams = rawVideoList[i].split("\\u0026"); - String videoURLParam, itagParam, sigParam; - videoURLParam = itagParam = sigParam = ""; + int extractionCount = 0; + for (String itag : itagPriority) { + if (parsedVideoMap.containsKey(itag) + && (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) { + logger.fine("adding video: " + parsedVideoMap.get(itag)); + addOutlink(uri, parsedVideoMap.get(itag), + org.archive.modules.extractor.LinkContext.EMBED_MISC, + org.archive.modules.extractor.Hop.EMBED); + extractionCount++; + } + } - for (String param : videoParams) { + // if itagPriority not specified, make sure we consider all discovered + // video urls + if (getItagPriority() == null || getItagPriority().isEmpty()) { + Iterator itagKeyIter = parsedVideoMap.keySet().iterator(); + while (itagKeyIter.hasNext() && (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) { + String itag = itagKeyIter.next(); + if (!KNOWN_ITAGS.contains(itag)) { + logger.warning("adding video (with unknown itag " + itag + + "): " + parsedVideoMap.get(itag)); + addOutlink(uri, parsedVideoMap.get(itag), + org.archive.modules.extractor.LinkContext.EMBED_MISC, + org.archive.modules.extractor.Hop.EMBED); + extractionCount++; + } + } + } + } - String[] keyValuePair = param.split("="); - if (keyValuePair.length != 2) { - logger.warning("Invalid Video Parameter: " + param); - continue; - } + private LinkedHashMap parseStreamMap(String streamMap) { + String[] rawVideoList = streamMap.split(","); + LinkedHashMap parsedVideoMap = new LinkedHashMap(); - if (keyValuePair[0].equals("url")) { - videoURLParam = keyValuePair[1]; - } - if (keyValuePair[0].equals("itag")) { - itagParam = keyValuePair[1]; - } - if (keyValuePair[0].equals("sig")) { - sigParam = keyValuePair[1]; - } - } + // Parse Video Map into itag,url pair + for (int i = 0; i < rawVideoList.length; i++) { + String[] videoParams = rawVideoList[i].split("\\u0026"); + String videoURLParam, itagParam, sigParam; + videoURLParam = itagParam = sigParam = ""; - if (videoURLParam.length() > 0 && itagParam.length() > 0 - && sigParam.length() > 0) { - try { - String fixupURL = URLDecoder.decode(videoURLParam - + "%26signature=" + sigParam, "UTF-8"); - parsedVideoMap.put(itagParam, fixupURL); - } catch (java.io.UnsupportedEncodingException e) { - logger.warning("Error decoding youtube video URL: " - + videoURLParam + "%26signature=" + sigParam); - } - } - } - return parsedVideoMap; - } + for (String param : videoParams) { + + String[] keyValuePair = param.split("="); + if (keyValuePair.length != 2) { + logger.warning("Invalid Video Parameter: " + param); + continue; + } + + if (keyValuePair[0].equals("url")) { + videoURLParam = keyValuePair[1]; + } + if (keyValuePair[0].equals("itag")) { + itagParam = keyValuePair[1]; + } + if (keyValuePair[0].equals("sig")) { + sigParam = keyValuePair[1]; + } + } + + if (videoURLParam.length() > 0 && itagParam.length() > 0 + && sigParam.length() > 0) { + try { + String fixupURL = URLDecoder.decode(videoURLParam + + "%26signature=" + sigParam, "UTF-8"); + parsedVideoMap.put(itagParam, fixupURL); + } catch (java.io.UnsupportedEncodingException e) { + logger.warning("Error decoding youtube video URL: " + + videoURLParam + "%26signature=" + sigParam); + } + } + } + return parsedVideoMap; + } } \ No newline at end of file diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java index 9f1a5fb7..90e61136 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBase.java @@ -18,15 +18,9 @@ */ package org.archive.modules.recrawl.hbase; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY; -import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG; - -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; -import java.util.logging.LogManager; import java.util.logging.Logger; import org.apache.hadoop.conf.Configuration; @@ -35,9 +29,6 @@ import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HConnectionManager; -import org.archive.modules.CrawlURI; -import org.archive.modules.recrawl.FetchHistoryProcessor; -import org.archive.net.UURIFactory; import org.springframework.context.Lifecycle; /** @@ -47,71 +38,71 @@ import org.springframework.context.Lifecycle; */ public class HBase implements Lifecycle { - private static final Logger logger = - Logger.getLogger(HBase.class.getName()); + private static final Logger logger = + Logger.getLogger(HBase.class.getName()); - protected Configuration conf = null; + protected Configuration conf = null; - private Map properties; - - public Map getProperties() { - return properties; - } - - public void setProperties(Map properties) { - this.properties = properties; + private Map properties; - if (conf == null) { - conf = HBaseConfiguration.create(); - } - for (Entry entry: getProperties().entrySet()) { - conf.set(entry.getKey(), entry.getValue()); - } - } + public Map getProperties() { + return properties; + } - public synchronized Configuration configuration() { - if (conf == null) { - conf = HBaseConfiguration.create(); - } + public void setProperties(Map properties) { + this.properties = properties; - return conf; - } + if (conf == null) { + conf = HBaseConfiguration.create(); + } + for (Entry entry: getProperties().entrySet()) { + conf.set(entry.getKey(), entry.getValue()); + } + } - protected transient HBaseAdmin admin; + public synchronized Configuration configuration() { + if (conf == null) { + conf = HBaseConfiguration.create(); + } - public synchronized HBaseAdmin admin() throws MasterNotRunningException, ZooKeeperConnectionException { - if (admin == null) { - admin = new HBaseAdmin(configuration()); - } + return conf; + } - return admin; - } + protected transient HBaseAdmin admin; - @Override - public synchronized void stop() { - isRunning = false; - if (admin != null) { - try { - admin.close(); - } catch (IOException e) { - logger.warning("problem closing HBaseAdmin " + admin + " - " + e); - } - - admin = null; - } - if (conf != null) { - HConnectionManager.deleteConnection(conf, true); - } - } + public synchronized HBaseAdmin admin() throws MasterNotRunningException, ZooKeeperConnectionException { + if (admin == null) { + admin = new HBaseAdmin(configuration()); + } - protected transient boolean isRunning = false; - @Override - public boolean isRunning() { - return isRunning; - } + return admin; + } - @Override - public void start() { - isRunning = true; - } + @Override + public synchronized void stop() { + isRunning = false; + if (admin != null) { + try { + admin.close(); + } catch (IOException e) { + logger.warning("problem closing HBaseAdmin " + admin + " - " + e); + } + + admin = null; + } + if (conf != null) { + HConnectionManager.deleteConnection(conf, true); + } + } + + protected transient boolean isRunning = false; + @Override + public boolean isRunning() { + return isRunning; + } + + @Override + public void start() { + isRunning = true; + } } diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java index 33e78c57..41197418 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseContentDigestHistory.java @@ -43,159 +43,159 @@ public class HBaseContentDigestHistory extends AbstractContentDigestHistory impl private static final Logger logger = Logger.getLogger(HBaseContentDigestHistory.class.getName()); - - protected static final byte[] COLUMN_FAMILY = Bytes.toBytes("f"); - protected static final byte[] COLUMN = Bytes.toBytes("c"); - - protected static final BiMap JSON_KEYS_MAP = HashBiMap.create(); - static { - JSON_KEYS_MAP.put(A_CONTENT_DIGEST_COUNT, "c"); - JSON_KEYS_MAP.put(A_ORIGINAL_URL, "u"); - JSON_KEYS_MAP.put(A_WARC_RECORD_ID, "i"); - JSON_KEYS_MAP.put(A_WARC_FILENAME, "f"); - JSON_KEYS_MAP.put(A_WARC_FILE_OFFSET, "o"); - JSON_KEYS_MAP.put(A_ORIGINAL_DATE, "d"); - } - - protected HBaseTable table; - public void setTable(HBaseTable table) { - this.table = table; - } - protected boolean addColumnFamily = false; - public boolean getAddColumnFamily() { - return addColumnFamily; - } - /** - * Add the expected column family - * {@link HBasePersistProcessor#COLUMN_FAMILY} to the HBase table if the - * table doesn't already have it. - */ - public void setAddColumnFamily(boolean addColumnFamily) { - this.addColumnFamily = addColumnFamily; - } + protected static final byte[] COLUMN_FAMILY = Bytes.toBytes("f"); + protected static final byte[] COLUMN = Bytes.toBytes("c"); - protected int retryIntervalMs = 10*1000; - public int getRetryIntervalMs() { - return retryIntervalMs; - } - public void setRetryIntervalMs(int retryIntervalMs) { - this.retryIntervalMs = retryIntervalMs; - } + protected static final BiMap JSON_KEYS_MAP = HashBiMap.create(); + static { + JSON_KEYS_MAP.put(A_CONTENT_DIGEST_COUNT, "c"); + JSON_KEYS_MAP.put(A_ORIGINAL_URL, "u"); + JSON_KEYS_MAP.put(A_WARC_RECORD_ID, "i"); + JSON_KEYS_MAP.put(A_WARC_FILENAME, "f"); + JSON_KEYS_MAP.put(A_WARC_FILE_OFFSET, "o"); + JSON_KEYS_MAP.put(A_ORIGINAL_DATE, "d"); + } - protected int maxTries = 1; - public int getMaxTries() { - return maxTries; - } - public void setMaxTries(int maxTries) { - this.maxTries = maxTries; - } + protected HBaseTable table; + public void setTable(HBaseTable table) { + this.table = table; + } - protected synchronized void addColumnFamily() { - try { - HTableDescriptor oldDesc = table.getHtableDescriptor(); - if (oldDesc.getFamily(COLUMN_FAMILY) == null) { - HTableDescriptor newDesc = new HTableDescriptor(oldDesc); - newDesc.addFamily(new HColumnDescriptor(COLUMN_FAMILY)); - logger.info("table does not yet have expected column family, modifying descriptor to " + newDesc); - HBaseAdmin hbaseAdmin = table.getHbase().admin(); - hbaseAdmin.disableTable(table.getName()); - hbaseAdmin.modifyTable(Bytes.toBytes(table.getName()), newDesc); - hbaseAdmin.enableTable(table.getName()); - } - } catch (IOException e) { - logger.warning("problem adding column family: " + e); - } - } + protected boolean addColumnFamily = false; + public boolean getAddColumnFamily() { + return addColumnFamily; + } + /** + * Add the expected column family + * {@link HBasePersistProcessor#COLUMN_FAMILY} to the HBase table if the + * table doesn't already have it. + */ + public void setAddColumnFamily(boolean addColumnFamily) { + this.addColumnFamily = addColumnFamily; + } - private boolean isRunning; - @Override - public void start() { - // add column family here to avoid disabling table while another - // ToeThread is trying to use it - if (getAddColumnFamily()) { - addColumnFamily(); - } - this.isRunning = true; - } - @Override - public void stop() { - this.isRunning = false; - } - @Override - public boolean isRunning() { - return isRunning; - } - - @Override - public void load(CrawlURI curi) { - // make this call in all cases so that the value is initialized and - // WARCWriterProcessor knows it should put the info in there - HashMap contentDigestHistory = curi.getContentDigestHistory(); + protected int retryIntervalMs = 10*1000; + public int getRetryIntervalMs() { + return retryIntervalMs; + } + public void setRetryIntervalMs(int retryIntervalMs) { + this.retryIntervalMs = retryIntervalMs; + } - byte[] key = Bytes.toBytes(persistKeyFor(curi)); - Result hbaseResult = tryHbaseGet(curi, new Get(key)); + protected int maxTries = 1; + public int getMaxTries() { + return maxTries; + } + public void setMaxTries(int maxTries) { + this.maxTries = maxTries; + } - if (hbaseResult != null) { - Map loadedHistory = parseHbaseResult(curi, hbaseResult); - - if (loadedHistory != null) { - if (logger.isLoggable(Level.FINER)) { - logger.finer("loaded history by digest " + persistKeyFor(curi) - + " for uri " + curi + " - " + loadedHistory); - } - contentDigestHistory.putAll(loadedHistory); - } - } - } + protected synchronized void addColumnFamily() { + try { + HTableDescriptor oldDesc = table.getHtableDescriptor(); + if (oldDesc.getFamily(COLUMN_FAMILY) == null) { + HTableDescriptor newDesc = new HTableDescriptor(oldDesc); + newDesc.addFamily(new HColumnDescriptor(COLUMN_FAMILY)); + logger.info("table does not yet have expected column family, modifying descriptor to " + newDesc); + HBaseAdmin hbaseAdmin = table.getHbase().admin(); + hbaseAdmin.disableTable(table.getName()); + hbaseAdmin.modifyTable(Bytes.toBytes(table.getName()), newDesc); + hbaseAdmin.enableTable(table.getName()); + } + } catch (IOException e) { + logger.warning("problem adding column family: " + e); + } + } - protected Result tryHbaseGet(CrawlURI curi, Get hbaseGet) { - try { - return table.get(hbaseGet); - } catch (IOException e) { - logger.warning("problem retrieving persist data from hbase, proceeding without, for digest " + persistKeyFor(curi) + " uri " + curi + " - " + e); - return null; - } - } + private boolean isRunning; + @Override + public void start() { + // add column family here to avoid disabling table while another + // ToeThread is trying to use it + if (getAddColumnFamily()) { + addColumnFamily(); + } + this.isRunning = true; + } + @Override + public void stop() { + this.isRunning = false; + } + @Override + public boolean isRunning() { + return isRunning; + } - protected Map parseHbaseResult(CrawlURI curi, Result hbaseResult) { - HashMap loadedHistory = null; - // no data for uri is indicated by empty Result - if (!hbaseResult.isEmpty()) { - byte[] jsonBytes = hbaseResult.getValue(COLUMN_FAMILY, COLUMN); - if (jsonBytes != null) { - JSONObject json = null; - try { - json = new JSONObject(Bytes.toString(jsonBytes)); - loadedHistory = new HashMap(); - @SuppressWarnings("unchecked") - Iterator keyIter = json.keys(); - while (keyIter.hasNext()) { - String jsonKey = keyIter.next(); - Object jsonValue = json.get(jsonKey); - String historyMapKey = JSON_KEYS_MAP.inverse().get(jsonKey); - if (historyMapKey == null) { - logger.warning("unknown key \"" + jsonKey + "\" found in hbase json for digest " + persistKeyFor(curi)); - historyMapKey = jsonKey; - } - loadedHistory.put(historyMapKey, jsonValue); - } - } catch (JSONException e) { - logger.warning("problem parsing json for digest " + persistKeyFor(curi) + " uri " + curi + " - " + e); - } - } else { - // shouldn't happen? result.isEmpty() is normal case - logger.fine("[jsonBytes==null] no persist data for digest " + persistKeyFor(curi) + " uri " + curi); - } - } else { - logger.finest("[result.isEmpty()] no persist data for digest " + persistKeyFor(curi) + " uri " + curi); - } + @Override + public void load(CrawlURI curi) { + // make this call in all cases so that the value is initialized and + // WARCWriterProcessor knows it should put the info in there + HashMap contentDigestHistory = curi.getContentDigestHistory(); - return loadedHistory; - } - - @Override - public void store(CrawlURI curi) { + byte[] key = Bytes.toBytes(persistKeyFor(curi)); + Result hbaseResult = tryHbaseGet(curi, new Get(key)); + + if (hbaseResult != null) { + Map loadedHistory = parseHbaseResult(curi, hbaseResult); + + if (loadedHistory != null) { + if (logger.isLoggable(Level.FINER)) { + logger.finer("loaded history by digest " + persistKeyFor(curi) + + " for uri " + curi + " - " + loadedHistory); + } + contentDigestHistory.putAll(loadedHistory); + } + } + } + + protected Result tryHbaseGet(CrawlURI curi, Get hbaseGet) { + try { + return table.get(hbaseGet); + } catch (IOException e) { + logger.warning("problem retrieving persist data from hbase, proceeding without, for digest " + persistKeyFor(curi) + " uri " + curi + " - " + e); + return null; + } + } + + protected Map parseHbaseResult(CrawlURI curi, Result hbaseResult) { + HashMap loadedHistory = null; + // no data for uri is indicated by empty Result + if (!hbaseResult.isEmpty()) { + byte[] jsonBytes = hbaseResult.getValue(COLUMN_FAMILY, COLUMN); + if (jsonBytes != null) { + JSONObject json = null; + try { + json = new JSONObject(Bytes.toString(jsonBytes)); + loadedHistory = new HashMap(); + @SuppressWarnings("unchecked") + Iterator keyIter = json.keys(); + while (keyIter.hasNext()) { + String jsonKey = keyIter.next(); + Object jsonValue = json.get(jsonKey); + String historyMapKey = JSON_KEYS_MAP.inverse().get(jsonKey); + if (historyMapKey == null) { + logger.warning("unknown key \"" + jsonKey + "\" found in hbase json for digest " + persistKeyFor(curi)); + historyMapKey = jsonKey; + } + loadedHistory.put(historyMapKey, jsonValue); + } + } catch (JSONException e) { + logger.warning("problem parsing json for digest " + persistKeyFor(curi) + " uri " + curi + " - " + e); + } + } else { + // shouldn't happen? result.isEmpty() is normal case + logger.fine("[jsonBytes==null] no persist data for digest " + persistKeyFor(curi) + " uri " + curi); + } + } else { + logger.finest("[result.isEmpty()] no persist data for digest " + persistKeyFor(curi) + " uri " + curi); + } + + return loadedHistory; + } + + @Override + public void store(CrawlURI curi) { if (!curi.hasContentDigestHistory() || curi.getContentDigestHistory().isEmpty()) { logger.warning("not saving empty content digest history (do you " @@ -208,69 +208,69 @@ public class HBaseContentDigestHistory extends AbstractContentDigestHistory impl + " for uri " + curi + " - " + curi.getContentDigestHistory()); } - - Put hbasePut = createHbasePut(curi); - tryHbasePut(curi, hbasePut); - } - protected void tryHbasePut(CrawlURI curi, Put p) { - int tryCount = 0; - do { - tryCount++; - try { - table.put(p); - return; - } catch (RetriesExhaustedWithDetailsException e) { - if (e.getCause(0) instanceof NoSuchColumnFamilyException && getAddColumnFamily()) { - addColumnFamily(); - tryCount--; - } else { - logger.warning("put failed " + "(try " + tryCount + " of " - + getMaxTries() + ")" + " for " + curi + " - " + e); - } - } catch (IOException e) { - logger.warning("put failed " + "(try " + tryCount + " of " - + getMaxTries() + ")" + " for " + curi + " - " + e); - } catch (NullPointerException e) { - // HTable.put() throws NullPointerException while connection is lost. - logger.warning("put failed " + "(try " + tryCount + " of " - + getMaxTries() + ")" + " for " + curi + " - " + e); - } + Put hbasePut = createHbasePut(curi); + tryHbasePut(curi, hbasePut); + } - if (tryCount > 0 && tryCount < getMaxTries() && isRunning()) { - try { - Thread.sleep(getRetryIntervalMs()); - } catch (InterruptedException ex) { - logger.warning("thread interrupted. aborting retry for " + curi); - return; - } - } - } while (tryCount < getMaxTries() && isRunning()); + protected void tryHbasePut(CrawlURI curi, Put p) { + int tryCount = 0; + do { + tryCount++; + try { + table.put(p); + return; + } catch (RetriesExhaustedWithDetailsException e) { + if (e.getCause(0) instanceof NoSuchColumnFamilyException && getAddColumnFamily()) { + addColumnFamily(); + tryCount--; + } else { + logger.warning("put failed " + "(try " + tryCount + " of " + + getMaxTries() + ")" + " for " + curi + " - " + e); + } + } catch (IOException e) { + logger.warning("put failed " + "(try " + tryCount + " of " + + getMaxTries() + ")" + " for " + curi + " - " + e); + } catch (NullPointerException e) { + // HTable.put() throws NullPointerException while connection is lost. + logger.warning("put failed " + "(try " + tryCount + " of " + + getMaxTries() + ")" + " for " + curi + " - " + e); + } - if (isRunning()) { - logger.warning("giving up after " + tryCount + " tries on put for " + curi); - } - } - - protected Put createHbasePut(CrawlURI curi) { - byte[] key = Bytes.toBytes(persistKeyFor(curi)); - Put hbasePut = new Put(key); - try { - JSONObject json = new JSONObject(); - for (Entry entry: curi.getContentDigestHistory().entrySet()) { - String jsonKey = JSON_KEYS_MAP.get(entry.getKey()); - if (jsonKey == null) { - logger.warning("unknown key \"" + entry.getKey() + "\" found in content digest history map for " + curi); - jsonKey = entry.getKey(); - } - json.put(jsonKey, entry.getValue()); - } - hbasePut.add(COLUMN_FAMILY, COLUMN, Bytes.toBytes(json.toString())); - } catch (JSONException e) { - // should not happen - all values are either primitive or String. - logger.log(Level.SEVERE, "problem creating json object for digest " + persistKeyFor(curi) + " uri " + curi, e); - } - return hbasePut; - } + if (tryCount > 0 && tryCount < getMaxTries() && isRunning()) { + try { + Thread.sleep(getRetryIntervalMs()); + } catch (InterruptedException ex) { + logger.warning("thread interrupted. aborting retry for " + curi); + return; + } + } + } while (tryCount < getMaxTries() && isRunning()); + + if (isRunning()) { + logger.warning("giving up after " + tryCount + " tries on put for " + curi); + } + } + + protected Put createHbasePut(CrawlURI curi) { + byte[] key = Bytes.toBytes(persistKeyFor(curi)); + Put hbasePut = new Put(key); + try { + JSONObject json = new JSONObject(); + for (Entry entry: curi.getContentDigestHistory().entrySet()) { + String jsonKey = JSON_KEYS_MAP.get(entry.getKey()); + if (jsonKey == null) { + logger.warning("unknown key \"" + entry.getKey() + "\" found in content digest history map for " + curi); + jsonKey = entry.getKey(); + } + json.put(jsonKey, entry.getValue()); + } + hbasePut.add(COLUMN_FAMILY, COLUMN, Bytes.toBytes(json.toString())); + } catch (JSONException e) { + // should not happen - all values are either primitive or String. + logger.log(Level.SEVERE, "problem creating json object for digest " + persistKeyFor(curi) + " uri " + curi, e); + } + return hbasePut; + } } diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistLoadProcessor.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistLoadProcessor.java index 63d6c193..d1da2e87 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistLoadProcessor.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistLoadProcessor.java @@ -40,60 +40,60 @@ import org.archive.modules.recrawl.RecrawlAttributeConstants; * @contributor kenji */ public class HBasePersistLoadProcessor extends HBasePersistProcessor { - private static final Logger logger = - Logger.getLogger(HBasePersistLoadProcessor.class.getName()); + private static final Logger logger = + Logger.getLogger(HBasePersistLoadProcessor.class.getName()); - @SuppressWarnings("unchecked") - protected static Map[] getFetchHistory(CrawlURI uri) { - Map data = uri.getData(); - Map[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); - if (history == null) { - // only the first element is used by FetchHTTP, WarcWriterProcessor etc. - // FetchHistoryProcessor casts history to HashMap[]. So it must be new HashMap[1]. - history = new HashMap[2]; - data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history); - } - return history; - } + @SuppressWarnings("unchecked") + protected static Map[] getFetchHistory(CrawlURI uri) { + Map data = uri.getData(); + Map[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); + if (history == null) { + // only the first element is used by FetchHTTP, WarcWriterProcessor etc. + // FetchHistoryProcessor casts history to HashMap[]. So it must be new HashMap[1]. + history = new HashMap[2]; + data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history); + } + return history; + } - @Override - protected ProcessResult innerProcessResult(CrawlURI uri) throws InterruptedException { - byte[] key = Bytes.toBytes(PersistProcessor.persistKeyFor(uri)); - Get g = new Get(key); - try { - Result r = table.get(g); - // no data for uri is indicated by empty Result - if (r.isEmpty()) { - if (logger.isLoggable(Level.FINE)) { - logger.fine(uri + ": "); - } - return ProcessResult.PROCEED; - } - schema.load(r, uri); - if (uri.getFetchStatus() < 0) { - return ProcessResult.FINISH; - } - } catch (IOException e) { - logger.warning("problem retrieving persist data from hbase, proceeding without, for " + uri + " - " + e); - } - return ProcessResult.PROCEED; - } + @Override + protected ProcessResult innerProcessResult(CrawlURI uri) throws InterruptedException { + byte[] key = Bytes.toBytes(PersistProcessor.persistKeyFor(uri)); + Get g = new Get(key); + try { + Result r = table.get(g); + // no data for uri is indicated by empty Result + if (r.isEmpty()) { + if (logger.isLoggable(Level.FINE)) { + logger.fine(uri + ": "); + } + return ProcessResult.PROCEED; + } + schema.load(r, uri); + if (uri.getFetchStatus() < 0) { + return ProcessResult.FINISH; + } + } catch (IOException e) { + logger.warning("problem retrieving persist data from hbase, proceeding without, for " + uri + " - " + e); + } + return ProcessResult.PROCEED; + } - /** - * unused. - */ - @Override - protected void innerProcess(CrawlURI uri) throws InterruptedException { - } + /** + * unused. + */ + @Override + protected void innerProcess(CrawlURI uri) throws InterruptedException { + } - @Override - protected boolean shouldProcess(CrawlURI uri) { - // TODO: we want deduplicate robots.txt, too. - //if (uri.isPrerequisite()) return false; - String scheme = uri.getUURI().getScheme(); - if (!(scheme.equals("http") || scheme.equals("https") || scheme.equals("ftp"))) { - return false; - } - return true; - } + @Override + protected boolean shouldProcess(CrawlURI uri) { + // TODO: we want deduplicate robots.txt, too. + //if (uri.isPrerequisite()) return false; + String scheme = uri.getUURI().getScheme(); + if (!(scheme.equals("http") || scheme.equals("https") || scheme.equals("ftp"))) { + return false; + } + return true; + } } diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistProcessor.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistProcessor.java index efdafe72..490cbbfb 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistProcessor.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistProcessor.java @@ -19,21 +19,21 @@ import org.springframework.beans.factory.annotation.Required; */ public abstract class HBasePersistProcessor extends AbstractPersistProcessor { - protected HBaseTable table; - public void setTable(HBaseTable table) { - this.table = table; - } + protected HBaseTable table; + public void setTable(HBaseTable table) { + this.table = table; + } - protected RecrawlDataSchema schema; - public RecrawlDataSchema getSchema() { - return schema; - } - @Required - public void setSchema(RecrawlDataSchema schema) { - this.schema = schema; - } + protected RecrawlDataSchema schema; + public RecrawlDataSchema getSchema() { + return schema; + } + @Required + public void setSchema(RecrawlDataSchema schema) { + this.schema = schema; + } - public HBasePersistProcessor() { - super(); - } + public HBasePersistProcessor() { + super(); + } } diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistStoreProcessor.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistStoreProcessor.java index e5a6565b..46c05be4 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistStoreProcessor.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBasePersistStoreProcessor.java @@ -36,98 +36,98 @@ import org.archive.modules.recrawl.RecrawlAttributeConstants; * @contributor kenji */ public class HBasePersistStoreProcessor extends HBasePersistProcessor implements FetchStatusCodes, RecrawlAttributeConstants { - private static final Logger logger = Logger.getLogger(HBasePersistStoreProcessor.class.getName()); + private static final Logger logger = Logger.getLogger(HBasePersistStoreProcessor.class.getName()); - protected boolean addColumnFamily = false; - public boolean getAddColumnFamily() { - return addColumnFamily; - } - /** - * Add the expected column family - * {@link HBasePersistProcessor#COLUMN_FAMILY} to the HBase table if the - * table doesn't already have it. - */ - public void setAddColumnFamily(boolean addColumnFamily) { - this.addColumnFamily = addColumnFamily; - } + protected boolean addColumnFamily = false; + public boolean getAddColumnFamily() { + return addColumnFamily; + } + /** + * Add the expected column family + * {@link HBasePersistProcessor#COLUMN_FAMILY} to the HBase table if the + * table doesn't already have it. + */ + public void setAddColumnFamily(boolean addColumnFamily) { + this.addColumnFamily = addColumnFamily; + } - protected int retryIntervalMs = 10*1000; - public int getRetryIntervalMs() { - return retryIntervalMs; - } - public void setRetryIntervalMs(int retryIntervalMs) { - this.retryIntervalMs = retryIntervalMs; - } + protected int retryIntervalMs = 10*1000; + public int getRetryIntervalMs() { + return retryIntervalMs; + } + public void setRetryIntervalMs(int retryIntervalMs) { + this.retryIntervalMs = retryIntervalMs; + } - protected int maxTries = 1; - public int getMaxTries() { - return maxTries; - } - public void setMaxTries(int maxTries) { - this.maxTries = maxTries; - } + protected int maxTries = 1; + public int getMaxTries() { + return maxTries; + } + public void setMaxTries(int maxTries) { + this.maxTries = maxTries; + } - protected synchronized void addColumnFamily() { - try { - HTableDescriptor oldDesc = table.getHtableDescriptor(); - byte[] columnFamily = Bytes.toBytes(schema.getColumnFamily()); - if (oldDesc.getFamily(columnFamily) == null) { - HTableDescriptor newDesc = new HTableDescriptor(oldDesc); - newDesc.addFamily(new HColumnDescriptor(columnFamily)); - logger.info("table does not yet have expected column family, modifying descriptor to " + newDesc); - HBaseAdmin hbaseAdmin = table.getHbase().admin(); - hbaseAdmin.disableTable(table.getName()); - hbaseAdmin.modifyTable(Bytes.toBytes(table.getName()), newDesc); - hbaseAdmin.enableTable(table.getName()); - } - } catch (IOException e) { - logger.warning("problem adding column family: " + e); - } - } + protected synchronized void addColumnFamily() { + try { + HTableDescriptor oldDesc = table.getHtableDescriptor(); + byte[] columnFamily = Bytes.toBytes(schema.getColumnFamily()); + if (oldDesc.getFamily(columnFamily) == null) { + HTableDescriptor newDesc = new HTableDescriptor(oldDesc); + newDesc.addFamily(new HColumnDescriptor(columnFamily)); + logger.info("table does not yet have expected column family, modifying descriptor to " + newDesc); + HBaseAdmin hbaseAdmin = table.getHbase().admin(); + hbaseAdmin.disableTable(table.getName()); + hbaseAdmin.modifyTable(Bytes.toBytes(table.getName()), newDesc); + hbaseAdmin.enableTable(table.getName()); + } + } catch (IOException e) { + logger.warning("problem adding column family: " + e); + } + } - @Override - protected void innerProcess(CrawlURI uri) { - Put p = schema.createPut(uri); - int tryCount = 0; - do { - tryCount++; - try { - table.put(p); - return; - } catch (RetriesExhaustedWithDetailsException e) { - if (e.getCause(0) instanceof NoSuchColumnFamilyException && getAddColumnFamily()) { - addColumnFamily(); - tryCount--; - } else { - logger.warning("put failed " + "(try " + tryCount + " of " - + getMaxTries() + ")" + " for " + uri + " - " + e); - } - } catch (IOException e) { - logger.warning("put failed " + "(try " + tryCount + " of " - + getMaxTries() + ")" + " for " + uri + " - " + e); - } catch (NullPointerException e) { - // HTable.put() throws NullPointerException while connection is lost. - logger.warning("put failed " + "(try " + tryCount + " of " - + getMaxTries() + ")" + " for " + uri + " - " + e); - } + @Override + protected void innerProcess(CrawlURI uri) { + Put p = schema.createPut(uri); + int tryCount = 0; + do { + tryCount++; + try { + table.put(p); + return; + } catch (RetriesExhaustedWithDetailsException e) { + if (e.getCause(0) instanceof NoSuchColumnFamilyException && getAddColumnFamily()) { + addColumnFamily(); + tryCount--; + } else { + logger.warning("put failed " + "(try " + tryCount + " of " + + getMaxTries() + ")" + " for " + uri + " - " + e); + } + } catch (IOException e) { + logger.warning("put failed " + "(try " + tryCount + " of " + + getMaxTries() + ")" + " for " + uri + " - " + e); + } catch (NullPointerException e) { + // HTable.put() throws NullPointerException while connection is lost. + logger.warning("put failed " + "(try " + tryCount + " of " + + getMaxTries() + ")" + " for " + uri + " - " + e); + } - if (tryCount > 0 && tryCount < getMaxTries() && isRunning()) { - try { - Thread.sleep(getRetryIntervalMs()); - } catch (InterruptedException ex) { - logger.warning("thread interrupted. aborting retry for " + uri); - return; - } - } - } while (tryCount < getMaxTries() && isRunning()); + if (tryCount > 0 && tryCount < getMaxTries() && isRunning()) { + try { + Thread.sleep(getRetryIntervalMs()); + } catch (InterruptedException ex) { + logger.warning("thread interrupted. aborting retry for " + uri); + return; + } + } + } while (tryCount < getMaxTries() && isRunning()); - if (isRunning()) { - logger.warning("giving up after " + tryCount + " tries on put for " + uri); - } - } + if (isRunning()) { + logger.warning("giving up after " + tryCount + " tries on put for " + uri); + } + } - @Override - protected boolean shouldProcess(CrawlURI curi) { - return super.shouldStore(curi); - } + @Override + protected boolean shouldProcess(CrawlURI curi) { + return super.shouldStore(curi); + } } diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseTable.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseTable.java index b4469a2e..976c70a9 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseTable.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/HBaseTable.java @@ -38,116 +38,116 @@ import org.springframework.context.Lifecycle; */ public class HBaseTable implements Lifecycle { - private static final Logger logger = - Logger.getLogger(HBaseTable.class.getName()); + private static final Logger logger = + Logger.getLogger(HBaseTable.class.getName()); - // XXX this default doesn't really belong here if this is supposed to be a - // generic hbase table class - protected String name = PersistOnlineProcessor.URI_HISTORY_DBNAME; - public void setName(String name) { - this.name = name; - } - public String getName() { - return name; - } + // XXX this default doesn't really belong here if this is supposed to be a + // generic hbase table class + protected String name = PersistOnlineProcessor.URI_HISTORY_DBNAME; + public void setName(String name) { + this.name = name; + } + public String getName() { + return name; + } - protected HBase hbase = new HBase(); - public void setHbase(HBase hbase) { - this.hbase = hbase; - } - public HBase getHbase() { - return hbase; - } + protected HBase hbase = new HBase(); + public void setHbase(HBase hbase) { + this.hbase = hbase; + } + public HBase getHbase() { + return hbase; + } - protected boolean create = false; - public boolean getCreate() { - return create; - } - /** Create the named table if it doesn't exist. */ - public void setCreate(boolean create) { - this.create = create; - } + protected boolean create = false; + public boolean getCreate() { + return create; + } + /** Create the named table if it doesn't exist. */ + public void setCreate(boolean create) { + this.create = create; + } - protected HTablePool htablePool = null; + protected HTablePool htablePool = null; - public HBaseTable() { - } + public HBaseTable() { + } - protected synchronized HTablePool htablePool() throws IOException { - if (htablePool == null) { - // XXX maxSize = number of toe threads? - htablePool = new HTablePool(hbase.configuration(), - Integer.MAX_VALUE); - } + protected synchronized HTablePool htablePool() throws IOException { + if (htablePool == null) { + // XXX maxSize = number of toe threads? + htablePool = new HTablePool(hbase.configuration(), + Integer.MAX_VALUE); + } - return htablePool; - } + return htablePool; + } - public void put(Put p) throws IOException { - HTableInterface table = htablePool().getTable(name); - try { - table.put(p); - } finally { - htablePool().putTable(table); - // table.close(); // XXX hbase 0.92 - } - } + public void put(Put p) throws IOException { + HTableInterface table = htablePool().getTable(name); + try { + table.put(p); + } finally { + htablePool().putTable(table); + // table.close(); // XXX hbase 0.92 + } + } - public Result get(Get g) throws IOException { - HTableInterface table = htablePool().getTable(name); - try { - return table.get(g); - } finally { - htablePool().putTable(table); - // table.close(); // XXX hbase 0.92 - } - } + public Result get(Get g) throws IOException { + HTableInterface table = htablePool().getTable(name); + try { + return table.get(g); + } finally { + htablePool().putTable(table); + // table.close(); // XXX hbase 0.92 + } + } - public HTableDescriptor getHtableDescriptor() throws IOException { - HTableInterface table = htablePool().getTable(name); - try { - return table.getTableDescriptor(); - } finally { - htablePool().putTable(table); - } - } + public HTableDescriptor getHtableDescriptor() throws IOException { + HTableInterface table = htablePool().getTable(name); + try { + return table.getTableDescriptor(); + } finally { + htablePool().putTable(table); + } + } - protected boolean isRunning = false; + protected boolean isRunning = false; - @Override - public boolean isRunning() { - return isRunning; - } + @Override + public boolean isRunning() { + return isRunning; + } - @Override - public void start() { - try { - if (getCreate()) { - HBaseAdmin admin = hbase.admin(); - if (!admin.tableExists(name)) { - HTableDescriptor desc = new HTableDescriptor(name); - logger.info("hbase table '" + name + "' does not exist, creating it... " + desc); - admin.createTable(desc); - } - } - } catch (IOException e) { - logger.log(Level.SEVERE, "problem creating hbase table " + name, e); - } + @Override + public void start() { + try { + if (getCreate()) { + HBaseAdmin admin = hbase.admin(); + if (!admin.tableExists(name)) { + HTableDescriptor desc = new HTableDescriptor(name); + logger.info("hbase table '" + name + "' does not exist, creating it... " + desc); + admin.createTable(desc); + } + } + } catch (IOException e) { + logger.log(Level.SEVERE, "problem creating hbase table " + name, e); + } - isRunning = true; - } + isRunning = true; + } - @Override - public synchronized void stop() { - isRunning = false; - // org.apache.hadoop.io.IOUtils.closeStream(htablePool); // XXX hbase 0.92 - if (htablePool != null) { - try { - htablePool.close(); - } catch (IOException e) { - logger.warning("problem closing HTablePool " + htablePool + " - " + e); - } - htablePool = null; - } - } + @Override + public synchronized void stop() { + isRunning = false; + // org.apache.hadoop.io.IOUtils.closeStream(htablePool); // XXX hbase 0.92 + if (htablePool != null) { + try { + htablePool.close(); + } catch (IOException e) { + logger.warning("problem closing HTablePool " + htablePool + " - " + e); + } + htablePool = null; + } + } } diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/MultiColumnRecrawlDataSchema.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/MultiColumnRecrawlDataSchema.java index 13f616c8..af1fc420 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/MultiColumnRecrawlDataSchema.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/MultiColumnRecrawlDataSchema.java @@ -42,95 +42,95 @@ import org.archive.modules.recrawl.RecrawlAttributeConstants; * @contributor kenji */ public class MultiColumnRecrawlDataSchema extends RecrawlDataSchemaBase implements RecrawlDataSchema, RecrawlAttributeConstants { - static final Logger logger = Logger.getLogger(MultiColumnRecrawlDataSchema.class.getName()); + static final Logger logger = Logger.getLogger(MultiColumnRecrawlDataSchema.class.getName()); - public static final byte[] COLUMN_STATUS = Bytes.toBytes("s"); - public static final byte[] COLUMN_CONTENT_DIGEST = Bytes.toBytes("d"); - public static final byte[] COLUMN_ETAG = Bytes.toBytes("e"); - public static final byte[] COLUMN_LAST_MODIFIED = Bytes.toBytes("m"); + public static final byte[] COLUMN_STATUS = Bytes.toBytes("s"); + public static final byte[] COLUMN_CONTENT_DIGEST = Bytes.toBytes("d"); + public static final byte[] COLUMN_ETAG = Bytes.toBytes("e"); + public static final byte[] COLUMN_LAST_MODIFIED = Bytes.toBytes("m"); - /* (non-Javadoc) - * @see org.archive.modules.hq.recrawl.RecrawlDataSchema#createPut() - */ - public Put createPut(CrawlURI uri) { - byte[] uriBytes = Bytes.toBytes(uri.toString()); - byte[] key = uriBytes; - Put p = new Put(key); - String digest = uri.getContentDigestSchemeString(); - if (digest != null) { - p.add(columnFamily, COLUMN_CONTENT_DIGEST, Bytes.toBytes(digest)); - } - p.add(columnFamily, COLUMN_STATUS, Bytes.toBytes(Integer.toString(uri.getFetchStatus()))); - org.apache.commons.httpclient.HttpMethod method = uri.getHttpMethod(); - if (method != null) { - String etag = getHeaderValue(method, RecrawlAttributeConstants.A_ETAG_HEADER); - if (etag != null) { - // Etqg is usually quoted - if (etag.length() >= 2 && etag.charAt(0) == '"' && etag.charAt(etag.length() - 1) == '"') - etag = etag.substring(1, etag.length() - 1); - p.add(columnFamily, COLUMN_ETAG, Bytes.toBytes(etag)); - } - String lastmod = getHeaderValue(method, RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER); - if (lastmod != null) { - long lastmod_sec = parseHttpDate(lastmod); - if (lastmod_sec == 0) { - try { - lastmod_sec = uri.getFetchCompletedTime(); - } catch (NullPointerException ex) { - logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine()); - } - } - if (lastmod_sec != 0) - p.add(columnFamily, COLUMN_LAST_MODIFIED, Bytes.toBytes(lastmod_sec)); - } else { - try { - long completed = uri.getFetchCompletedTime(); - if (completed != 0) - p.add(columnFamily, COLUMN_LAST_MODIFIED, Bytes.toBytes(completed)); - } catch (NullPointerException ex) { - logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine()); - } - } - } - return p; - } + /* (non-Javadoc) + * @see org.archive.modules.hq.recrawl.RecrawlDataSchema#createPut() + */ + public Put createPut(CrawlURI uri) { + byte[] uriBytes = Bytes.toBytes(uri.toString()); + byte[] key = uriBytes; + Put p = new Put(key); + String digest = uri.getContentDigestSchemeString(); + if (digest != null) { + p.add(columnFamily, COLUMN_CONTENT_DIGEST, Bytes.toBytes(digest)); + } + p.add(columnFamily, COLUMN_STATUS, Bytes.toBytes(Integer.toString(uri.getFetchStatus()))); + org.apache.commons.httpclient.HttpMethod method = uri.getHttpMethod(); + if (method != null) { + String etag = getHeaderValue(method, RecrawlAttributeConstants.A_ETAG_HEADER); + if (etag != null) { + // Etqg is usually quoted + if (etag.length() >= 2 && etag.charAt(0) == '"' && etag.charAt(etag.length() - 1) == '"') + etag = etag.substring(1, etag.length() - 1); + p.add(columnFamily, COLUMN_ETAG, Bytes.toBytes(etag)); + } + String lastmod = getHeaderValue(method, RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER); + if (lastmod != null) { + long lastmod_sec = parseHttpDate(lastmod); + if (lastmod_sec == 0) { + try { + lastmod_sec = uri.getFetchCompletedTime(); + } catch (NullPointerException ex) { + logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine()); + } + } + if (lastmod_sec != 0) + p.add(columnFamily, COLUMN_LAST_MODIFIED, Bytes.toBytes(lastmod_sec)); + } else { + try { + long completed = uri.getFetchCompletedTime(); + if (completed != 0) + p.add(columnFamily, COLUMN_LAST_MODIFIED, Bytes.toBytes(completed)); + } catch (NullPointerException ex) { + logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine()); + } + } + } + return p; + } - /* (non-Javadoc) - * @see org.archive.modules.hq.recrawl.RecrawlDataSchema#load(java.util.Map, org.apache.hadoop.hbase.client.Result) - */ - public void load(Result result, CrawlURI curi) { - // check for "do-not-crawl" flag - any non-empty data tells not to crawl this - // URL. - byte[] nocrawl = result.getValue(columnFamily, COLUMN_NOCRAWL); - if (nocrawl != null && nocrawl.length > 0) { - // fetch status set to S_DEEMED_CHAFF, because this do-not-crawl flag - // is primarily intended for preventing crawler from stepping on traps. - curi.setFetchStatus(FetchStatusCodes.S_DEEMED_CHAFF); - curi.getAnnotations().add("nocrawl"); - return; - } + /* (non-Javadoc) + * @see org.archive.modules.hq.recrawl.RecrawlDataSchema#load(java.util.Map, org.apache.hadoop.hbase.client.Result) + */ + public void load(Result result, CrawlURI curi) { + // check for "do-not-crawl" flag - any non-empty data tells not to crawl this + // URL. + byte[] nocrawl = result.getValue(columnFamily, COLUMN_NOCRAWL); + if (nocrawl != null && nocrawl.length > 0) { + // fetch status set to S_DEEMED_CHAFF, because this do-not-crawl flag + // is primarily intended for preventing crawler from stepping on traps. + curi.setFetchStatus(FetchStatusCodes.S_DEEMED_CHAFF); + curi.getAnnotations().add("nocrawl"); + return; + } - Map history = getFetchHistory(curi); - // FetchHTTP ignores history with status <= 0 - byte[] status = result.getValue(columnFamily, COLUMN_STATUS); - if (status != null) { - // Note that status is stored as integer text. It's typically three-chars - // that is less than 4-byte integer bits. - history.put(RecrawlAttributeConstants.A_STATUS, Integer.parseInt(Bytes.toString(status))); - byte[] etag = result.getValue(columnFamily, COLUMN_ETAG); - if (etag != null) { - history.put(RecrawlAttributeConstants.A_ETAG_HEADER, Bytes.toString(etag)); - } - byte[] lastmod = result.getValue(columnFamily, COLUMN_LAST_MODIFIED); - if (lastmod != null) { - long lastmod_sec = Bytes.toLong(lastmod); - history.put(RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER, formatHttpDate(lastmod_sec)); - } - byte[] digest = result.getValue(columnFamily, COLUMN_CONTENT_DIGEST); - if (digest != null) { - history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, Bytes.toString(digest)); - } - } - } + Map history = getFetchHistory(curi); + // FetchHTTP ignores history with status <= 0 + byte[] status = result.getValue(columnFamily, COLUMN_STATUS); + if (status != null) { + // Note that status is stored as integer text. It's typically three-chars + // that is less than 4-byte integer bits. + history.put(RecrawlAttributeConstants.A_STATUS, Integer.parseInt(Bytes.toString(status))); + byte[] etag = result.getValue(columnFamily, COLUMN_ETAG); + if (etag != null) { + history.put(RecrawlAttributeConstants.A_ETAG_HEADER, Bytes.toString(etag)); + } + byte[] lastmod = result.getValue(columnFamily, COLUMN_LAST_MODIFIED); + if (lastmod != null) { + long lastmod_sec = Bytes.toLong(lastmod); + history.put(RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER, formatHttpDate(lastmod_sec)); + } + byte[] digest = result.getValue(columnFamily, COLUMN_CONTENT_DIGEST); + if (digest != null) { + history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, Bytes.toString(digest)); + } + } + } } diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/RecrawlDataSchema.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/RecrawlDataSchema.java index 34509d1d..446f71c8 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/RecrawlDataSchema.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/RecrawlDataSchema.java @@ -27,7 +27,7 @@ import org.archive.modules.CrawlURI; * @contributor kenji */ public interface RecrawlDataSchema { - public String getColumnFamily(); - public Put createPut(CrawlURI uri); - public void load(Result result, CrawlURI curi); + public String getColumnFamily(); + public Put createPut(CrawlURI uri); + public void load(Result result, CrawlURI curi); } diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/RecrawlDataSchemaBase.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/RecrawlDataSchemaBase.java index acaf1893..e938b387 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/RecrawlDataSchemaBase.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/RecrawlDataSchemaBase.java @@ -25,81 +25,81 @@ import org.archive.modules.recrawl.RecrawlAttributeConstants; * @contributor kenji */ abstract public class RecrawlDataSchemaBase implements RecrawlDataSchema { - private static final Logger logger = Logger.getLogger(RecrawlDataSchemaBase.class.getName()); + private static final Logger logger = Logger.getLogger(RecrawlDataSchemaBase.class.getName()); - public static final byte[] DEFAULT_COLUMN_FAMILY = Bytes.toBytes("f"); - protected byte[] columnFamily = DEFAULT_COLUMN_FAMILY; + public static final byte[] DEFAULT_COLUMN_FAMILY = Bytes.toBytes("f"); + protected byte[] columnFamily = DEFAULT_COLUMN_FAMILY; - public static final byte[] COLUMN_NOCRAWL = Bytes.toBytes("z"); + public static final byte[] COLUMN_NOCRAWL = Bytes.toBytes("z"); - protected static String getHeaderValue(org.apache.commons.httpclient.HttpMethod method, String name) { - org.apache.commons.httpclient.Header header = method.getResponseHeader(name); - return header != null ? header.getValue() : null; - } + protected static String getHeaderValue(org.apache.commons.httpclient.HttpMethod method, String name) { + org.apache.commons.httpclient.Header header = method.getResponseHeader(name); + return header != null ? header.getValue() : null; + } - public void setColumnFamily(String colf) { - columnFamily = Bytes.toBytes(colf); - } + public void setColumnFamily(String colf) { + columnFamily = Bytes.toBytes(colf); + } - @Override - public String getColumnFamily() { - return Bytes.toString(columnFamily); - } + @Override + public String getColumnFamily() { + return Bytes.toString(columnFamily); + } - /** - * returns a Map to store recrawl data, which is properly stored in CrawlURI's - * fetch history array property ({@link RecrawlAttributeConstants#A_FETCH_HISTORY} member of {@link CrawlURI#getData()}.) - * if {@code uri} has no fetch history yet, it is first initialized with an array of length two so that - * {@linkplain FetchHistoryProcessor} do not need to reallocate it (this only works for historyLength == 2, though). - * @param uri CrawlURI from which fetch history is obtained. - * @return Map object for storing re-crawl data (never null). - * @see FetchHistoryProcessor - */ - @SuppressWarnings("unchecked") - protected static Map getFetchHistory(CrawlURI uri) { - Map data = uri.getData(); - Map[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); - if (history == null) { - // only the first element is used by FetchHTTP, WarcWriterProcessor etc. - // FetchHistoryProcessor casts history to HashMap[]. So it must be new HashMap[2], not Map[2] - history = new HashMap[2]; - history[0] = new HashMap(); - // no need to set history[1]. it would simply be discarded by FetchHistoryProcessor. - data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history); - } - return history[0]; - } + /** + * returns a Map to store recrawl data, which is properly stored in CrawlURI's + * fetch history array property ({@link RecrawlAttributeConstants#A_FETCH_HISTORY} member of {@link CrawlURI#getData()}.) + * if {@code uri} has no fetch history yet, it is first initialized with an array of length two so that + * {@linkplain FetchHistoryProcessor} do not need to reallocate it (this only works for historyLength == 2, though). + * @param uri CrawlURI from which fetch history is obtained. + * @return Map object for storing re-crawl data (never null). + * @see FetchHistoryProcessor + */ + @SuppressWarnings("unchecked") + protected static Map getFetchHistory(CrawlURI uri) { + Map data = uri.getData(); + Map[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY); + if (history == null) { + // only the first element is used by FetchHTTP, WarcWriterProcessor etc. + // FetchHistoryProcessor casts history to HashMap[]. So it must be new HashMap[2], not Map[2] + history = new HashMap[2]; + history[0] = new HashMap(); + // no need to set history[1]. it would simply be discarded by FetchHistoryProcessor. + data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history); + } + return history[0]; + } - public RecrawlDataSchemaBase() { - super(); - } + public RecrawlDataSchemaBase() { + super(); + } - protected static final DateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); + protected static final DateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); - /** - * converts time in HTTP Date format {@code dateStr} to seconds - * since epoch. - * @param dateStr time in HTTP Date format. - * @return seconds since epoch - */ - protected long parseHttpDate(String dateStr) { - synchronized (HTTP_DATE_FORMAT) { - try { - Date d = HTTP_DATE_FORMAT.parse(dateStr); - return d.getTime() / 1000; - } catch (ParseException ex) { - if (logger.isLoggable(Level.FINER)) - logger.fine("bad HTTP DATE: " + dateStr); - return 0; - } - } - } + /** + * converts time in HTTP Date format {@code dateStr} to seconds + * since epoch. + * @param dateStr time in HTTP Date format. + * @return seconds since epoch + */ + protected long parseHttpDate(String dateStr) { + synchronized (HTTP_DATE_FORMAT) { + try { + Date d = HTTP_DATE_FORMAT.parse(dateStr); + return d.getTime() / 1000; + } catch (ParseException ex) { + if (logger.isLoggable(Level.FINER)) + logger.fine("bad HTTP DATE: " + dateStr); + return 0; + } + } + } - protected String formatHttpDate(long time) { - synchronized (HTTP_DATE_FORMAT) { - // format is not thread safe either - return HTTP_DATE_FORMAT.format(new Date(time * 1000)); - } - } + protected String formatHttpDate(long time) { + synchronized (HTTP_DATE_FORMAT) { + // format is not thread safe either + return HTTP_DATE_FORMAT.format(new Date(time * 1000)); + } + } } diff --git a/contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleColumnJsonRecrawlDataSchema.java b/contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleColumnJsonRecrawlDataSchema.java index 8f2ebf71..757f7018 100644 --- a/contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleColumnJsonRecrawlDataSchema.java +++ b/contrib/src/main/java/org/archive/modules/recrawl/hbase/SingleColumnJsonRecrawlDataSchema.java @@ -45,124 +45,124 @@ import org.json.JSONObject; */ public class SingleColumnJsonRecrawlDataSchema extends RecrawlDataSchemaBase implements RecrawlDataSchema { - static final Logger logger = Logger.getLogger(SingleColumnJsonRecrawlDataSchema.class.getName()); + static final Logger logger = Logger.getLogger(SingleColumnJsonRecrawlDataSchema.class.getName()); - public static byte[] DEFAULT_COLUMN = Bytes.toBytes("r"); + public static byte[] DEFAULT_COLUMN = Bytes.toBytes("r"); - // JSON property names for re-crawl data properties - public static final String PROPERTY_STATUS = "s"; - public static final String PROPERTY_CONTENT_DIGEST = "d"; - public static final String PROPERTY_ETAG = "e"; - public static final String PROPERTY_LAST_MODIFIED = "m"; + // JSON property names for re-crawl data properties + public static final String PROPERTY_STATUS = "s"; + public static final String PROPERTY_CONTENT_DIGEST = "d"; + public static final String PROPERTY_ETAG = "e"; + public static final String PROPERTY_LAST_MODIFIED = "m"; - // SHA1 scheme is assumed. - public static final String CONTENT_DIGEST_SCHEME = "sha1:"; + // SHA1 scheme is assumed. + public static final String CONTENT_DIGEST_SCHEME = "sha1:"; - // single column for storing JSON of re-crawl data - protected byte[] column = DEFAULT_COLUMN; - public void setColumn(String column) { - this.column = Bytes.toBytes(column); - } - public String getColumn() { - return Bytes.toString(column); - } + // single column for storing JSON of re-crawl data + protected byte[] column = DEFAULT_COLUMN; + public void setColumn(String column) { + this.column = Bytes.toBytes(column); + } + public String getColumn() { + return Bytes.toString(column); + } - /* (non-Javadoc) - * @see org.archive.modules.hq.recrawl.RecrawlDataSchema#createPut(org.archive.modules.CrawlURI) - */ - public Put createPut(CrawlURI uri) { - byte[] key = Bytes.toBytes(PersistProcessor.persistKeyFor(uri)); - Put p = new Put(key); - JSONObject jo = new JSONObject(); - try { - // TODO should we post warning message when scheme != "sha1"? - String digest = uri.getContentDigestString(); - if (digest != null) { - jo.put(PROPERTY_CONTENT_DIGEST, digest); - } - jo.put(PROPERTY_STATUS, uri.getFetchStatus()); - HttpMethod method = uri.getHttpMethod(); - if (method != null) { - String etag = getHeaderValue(method, RecrawlAttributeConstants.A_ETAG_HEADER); - if (etag != null) { - // Etag is usually quoted - if (etag.length() >= 2 && etag.charAt(0) == '"' && etag.charAt(etag.length() - 1) == '"') - etag = etag.substring(1, etag.length() - 1); - jo.put(PROPERTY_ETAG, etag); - } - String lastmod = getHeaderValue(method, RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER); - if (lastmod != null) { - // XXX value is not used anywhere, should it be? - long lastmod_sec = parseHttpDate(lastmod); - if (lastmod_sec == 0) { - try { - lastmod_sec = uri.getFetchCompletedTime(); - } catch (NullPointerException ex) { - logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine()); - } - } - } else { - try { - long completed = uri.getFetchCompletedTime(); - if (completed != 0) - jo.put(PROPERTY_LAST_MODIFIED, completed); - } catch (NullPointerException ex) { - logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine()); - } - } - } - } catch (JSONException ex) { - // should not happen - all values are either primitive or String. - logger.log(Level.SEVERE, "JSON translation failed", ex); - } - p.add(columnFamily, column, Bytes.toBytes(jo.toString())); - return p; - } + /* (non-Javadoc) + * @see org.archive.modules.hq.recrawl.RecrawlDataSchema#createPut(org.archive.modules.CrawlURI) + */ + public Put createPut(CrawlURI uri) { + byte[] key = Bytes.toBytes(PersistProcessor.persistKeyFor(uri)); + Put p = new Put(key); + JSONObject jo = new JSONObject(); + try { + // TODO should we post warning message when scheme != "sha1"? + String digest = uri.getContentDigestString(); + if (digest != null) { + jo.put(PROPERTY_CONTENT_DIGEST, digest); + } + jo.put(PROPERTY_STATUS, uri.getFetchStatus()); + HttpMethod method = uri.getHttpMethod(); + if (method != null) { + String etag = getHeaderValue(method, RecrawlAttributeConstants.A_ETAG_HEADER); + if (etag != null) { + // Etag is usually quoted + if (etag.length() >= 2 && etag.charAt(0) == '"' && etag.charAt(etag.length() - 1) == '"') + etag = etag.substring(1, etag.length() - 1); + jo.put(PROPERTY_ETAG, etag); + } + String lastmod = getHeaderValue(method, RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER); + if (lastmod != null) { + // XXX value is not used anywhere, should it be? + long lastmod_sec = parseHttpDate(lastmod); + if (lastmod_sec == 0) { + try { + lastmod_sec = uri.getFetchCompletedTime(); + } catch (NullPointerException ex) { + logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine()); + } + } + } else { + try { + long completed = uri.getFetchCompletedTime(); + if (completed != 0) + jo.put(PROPERTY_LAST_MODIFIED, completed); + } catch (NullPointerException ex) { + logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine()); + } + } + } + } catch (JSONException ex) { + // should not happen - all values are either primitive or String. + logger.log(Level.SEVERE, "JSON translation failed", ex); + } + p.add(columnFamily, column, Bytes.toBytes(jo.toString())); + return p; + } - /* (non-Javadoc) - * @see org.archive.modules.hq.recrawl.RecrawlDataSchema#load(org.apache.hadoop.hbase.client.Result) - */ - public void load(Result result, CrawlURI curi) { - // check for "do-not-crawl" flag - any non-empty data tells not to crawl this - // URL. - byte[] nocrawl = result.getValue(columnFamily, COLUMN_NOCRAWL); - if (nocrawl != null && nocrawl.length > 0) { - // fetch status set to S_DEEMED_CHAFF, because this do-not-crawl flag - // is primarily intended for preventing crawler from stepping on traps. - curi.setFetchStatus(FetchStatusCodes.S_DEEMED_CHAFF); - curi.getAnnotations().add("nocrawl"); - return; - } + /* (non-Javadoc) + * @see org.archive.modules.hq.recrawl.RecrawlDataSchema#load(org.apache.hadoop.hbase.client.Result) + */ + public void load(Result result, CrawlURI curi) { + // check for "do-not-crawl" flag - any non-empty data tells not to crawl this + // URL. + byte[] nocrawl = result.getValue(columnFamily, COLUMN_NOCRAWL); + if (nocrawl != null && nocrawl.length > 0) { + // fetch status set to S_DEEMED_CHAFF, because this do-not-crawl flag + // is primarily intended for preventing crawler from stepping on traps. + curi.setFetchStatus(FetchStatusCodes.S_DEEMED_CHAFF); + curi.getAnnotations().add("nocrawl"); + return; + } - Map history = getFetchHistory(curi); - byte[] jsonBytes = result.getValue(columnFamily, column); - if (jsonBytes != null) { - JSONObject jo = null; - try { - jo = new JSONObject(Bytes.toString(jsonBytes)); - } catch (JSONException ex) { - logger.warning(String.format("JSON parsing failed for key %1s: %2s", - result.getRow(), ex.getMessage())); - } - if (jo != null) { - int status = jo.optInt(PROPERTY_STATUS, -1); - if (status >= 0) { - history.put(RecrawlAttributeConstants.A_STATUS, status); - } - String digest = jo.optString(PROPERTY_CONTENT_DIGEST); - if (digest != null) { - history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, CONTENT_DIGEST_SCHEME + digest); - } - String etag = jo.optString(PROPERTY_ETAG); - if (etag != null) { - history.put(RecrawlAttributeConstants.A_ETAG_HEADER, etag); - } - long lastmod = jo.optLong(PROPERTY_LAST_MODIFIED); - if (lastmod > 0) { - history.put(RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER, formatHttpDate(lastmod)); - } - } - } - } + Map history = getFetchHistory(curi); + byte[] jsonBytes = result.getValue(columnFamily, column); + if (jsonBytes != null) { + JSONObject jo = null; + try { + jo = new JSONObject(Bytes.toString(jsonBytes)); + } catch (JSONException ex) { + logger.warning(String.format("JSON parsing failed for key %1s: %2s", + result.getRow(), ex.getMessage())); + } + if (jo != null) { + int status = jo.optInt(PROPERTY_STATUS, -1); + if (status >= 0) { + history.put(RecrawlAttributeConstants.A_STATUS, status); + } + String digest = jo.optString(PROPERTY_CONTENT_DIGEST); + if (digest != null) { + history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, CONTENT_DIGEST_SCHEME + digest); + } + String etag = jo.optString(PROPERTY_ETAG); + if (etag != null) { + history.put(RecrawlAttributeConstants.A_ETAG_HEADER, etag); + } + long lastmod = jo.optLong(PROPERTY_LAST_MODIFIED); + if (lastmod > 0) { + history.put(RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER, formatHttpDate(lastmod)); + } + } + } + } } diff --git a/contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java b/contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java index 4503c84e..61030cd6 100644 --- a/contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java +++ b/contrib/src/main/java/org/archive/util/OneLineSimpleLayout.java @@ -7,48 +7,48 @@ import org.apache.log4j.spi.LoggingEvent; public class OneLineSimpleLayout extends Layout { - private OneLineSimpleLogger logger = new OneLineSimpleLogger(); - - @Override - public void activateOptions() { - } + private OneLineSimpleLogger logger = new OneLineSimpleLogger(); - @Override - public String format(LoggingEvent event) { - java.util.logging.Level level = convertLevel(event.getLevel()); - - LogRecord logRecord = new LogRecord(level, event.getMessage().toString()); - logRecord.setLoggerName(event.getLoggerName()); - logRecord.setMillis(event.getTimeStamp()); - logRecord.setSourceClassName(event.getLoggerName()); - logRecord.setSourceMethodName(event.getLocationInformation().getMethodName()); - logRecord.setThreadID((int) Thread.currentThread().getId()); - - return logger.format(logRecord); - } + @Override + public void activateOptions() { + } - protected java.util.logging.Level convertLevel(org.apache.log4j.Level log4jLevel) { - switch (log4jLevel.toInt()) { - case org.apache.log4j.Level.TRACE_INT: - return java.util.logging.Level.FINER; - case org.apache.log4j.Level.DEBUG_INT: - return java.util.logging.Level.FINE; - case org.apache.log4j.Level.INFO_INT: - return java.util.logging.Level.INFO; - case org.apache.log4j.Level.WARN_INT: - return java.util.logging.Level.WARNING; - case org.apache.log4j.Level.ERROR_INT: - return java.util.logging.Level.SEVERE; - case org.apache.log4j.Level.FATAL_INT: - return java.util.logging.Level.SEVERE; - default: - return java.util.logging.Level.ALL; - } - } + @Override + public String format(LoggingEvent event) { + java.util.logging.Level level = convertLevel(event.getLevel()); - @Override - public boolean ignoresThrowable() { - return true; - } + LogRecord logRecord = new LogRecord(level, event.getMessage().toString()); + logRecord.setLoggerName(event.getLoggerName()); + logRecord.setMillis(event.getTimeStamp()); + logRecord.setSourceClassName(event.getLoggerName()); + logRecord.setSourceMethodName(event.getLocationInformation().getMethodName()); + logRecord.setThreadID((int) Thread.currentThread().getId()); + + return logger.format(logRecord); + } + + protected java.util.logging.Level convertLevel(org.apache.log4j.Level log4jLevel) { + switch (log4jLevel.toInt()) { + case org.apache.log4j.Level.TRACE_INT: + return java.util.logging.Level.FINER; + case org.apache.log4j.Level.DEBUG_INT: + return java.util.logging.Level.FINE; + case org.apache.log4j.Level.INFO_INT: + return java.util.logging.Level.INFO; + case org.apache.log4j.Level.WARN_INT: + return java.util.logging.Level.WARNING; + case org.apache.log4j.Level.ERROR_INT: + return java.util.logging.Level.SEVERE; + case org.apache.log4j.Level.FATAL_INT: + return java.util.logging.Level.SEVERE; + default: + return java.util.logging.Level.ALL; + } + } + + @Override + public boolean ignoresThrowable() { + return true; + } } diff --git a/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java index 335be6f2..b2acbeb2 100644 --- a/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java +++ b/contrib/src/test/java/org/archive/modules/extractor/ExtractorYoutubeFormatStreamTest.java @@ -23,26 +23,26 @@ import org.archive.util.Recorder; public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { - protected static final String TEST_URI = "http://www.youtube.com/watch?v=_BFJN62hZp0"; - protected static final String TEST_RESOURCE_FILE_NAME = "ExtractorYoutubeFormatStream.txt"; - + protected static final String TEST_URI = "http://www.youtube.com/watch?v=_BFJN62hZp0"; + protected static final String TEST_RESOURCE_FILE_NAME = "ExtractorYoutubeFormatStream.txt"; + protected static final String[] EXPECTED_OUTLINKS_ALL = { - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=44&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=98D0AC4D1B545DE6D5C531A5DB7902877511632A.5700D588C7659DE8A8621EC5110CB638D0EF4A39", - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703", - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=43&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=2E2A7F8D5A61497159C2A2CFBB07F62B98062500.33826449E77B3A5F5FE40D028857064D7313D60A", - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21", - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=18&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=9534F06E230AFD98894138B4A5D6DF75D11BC316.A488BE98A3EADBE84BA4A8BDA61E12EB6CA7BB85", - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=5&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=CA92F947487449BAB3D360E565331F5BE50D2134.8EC1CB03A41B459227D84F34D2C2AB7BC2BA95A0", - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=36&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=793FD335B7B7EE9B77B457DA951DAED704FFC363.1B6AC609D71AB3EE17BDA911E2761A87C79A080F", - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=17&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=06294DA91D0E1FE3C7DF7AD9D133EC95939B8D70.731CE14A56E9C61E4ABE43110CE7515C5F42C66B" + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=44&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=98D0AC4D1B545DE6D5C531A5DB7902877511632A.5700D588C7659DE8A8621EC5110CB638D0EF4A39", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=43&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=2E2A7F8D5A61497159C2A2CFBB07F62B98062500.33826449E77B3A5F5FE40D028857064D7313D60A", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=18&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=9534F06E230AFD98894138B4A5D6DF75D11BC316.A488BE98A3EADBE84BA4A8BDA61E12EB6CA7BB85", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=5&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=CA92F947487449BAB3D360E565331F5BE50D2134.8EC1CB03A41B459227D84F34D2C2AB7BC2BA95A0", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=36&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=793FD335B7B7EE9B77B457DA951DAED704FFC363.1B6AC609D71AB3EE17BDA911E2761A87C79A080F", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=17&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=06294DA91D0E1FE3C7DF7AD9D133EC95939B8D70.731CE14A56E9C61E4ABE43110CE7515C5F42C66B" }; protected static final String[] EXPECTED_OUTLINKS_SUBSET = { - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21" + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21" }; - + protected static final String[] EXPECTED_SINGLE_DEFAULT_OUTLINK = { - "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703", + "http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703", }; @Override @@ -52,71 +52,71 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { e.setLoggerModule(ulm); return e; } - + public void testAllInItagPriority() throws Exception { CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); - + List itagPriorityList = Arrays.asList("44", "35", "43", "34", "18", "5", "36", "17"); extractor().setItagPriority(itagPriorityList); extractor().setExtractLimit(10); extractor.process(testUri); - Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL); + Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL); assertEquals(expected, testUri.getOutLinks()); } public void testAllNoPriority() throws Exception { CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); - + extractor().setExtractLimit(0); extractor.process(testUri); - Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL); + Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL); assertEquals(expected, testUri.getOutLinks()); } - // test that only itags in the priority list are extracted, even though - // extract limit is large + // test that only itags in the priority list are extracted, even though + // extract limit is large public void testOnlyInItagPriorityBigLimit() throws Exception { CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); - + List itagPriorityList = Arrays.asList("44", "35", "43"); extractor().setItagPriority(itagPriorityList); extractor().setExtractLimit(10); extractor.process(testUri); - assertEquals(3, testUri.getOutLinks().size()); + assertEquals(3, testUri.getOutLinks().size()); } - // test that only itags in the priority list are extracted, even though - // extract limit is unset + // test that only itags in the priority list are extracted, even though + // extract limit is unset public void testOnlyInItagPriorityNoLimit() throws Exception { CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); - + List itagPriorityList = Arrays.asList("44", "35", "43"); extractor().setItagPriority(itagPriorityList); extractor().setExtractLimit(0); extractor.process(testUri); - assertEquals(3, testUri.getOutLinks().size()); + assertEquals(3, testUri.getOutLinks().size()); } - + public void testNoPriorityWithLimit() throws InterruptedException, URIException, UnsupportedEncodingException, IOException { CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); - + extractor().setExtractLimit(4); extractor.process(testUri); - assertEquals(4, testUri.getOutLinks().size()); + assertEquals(4, testUri.getOutLinks().size()); } - + public void testDontExtract() throws URIException, UnsupportedEncodingException, IOException, InterruptedException { - // not a youtube watch url so shouldProcess() will return false + // not a youtube watch url so shouldProcess() will return false CrawlURI testUri = createTestUri("http://archive.org/watch?w=blah", TEST_RESOURCE_FILE_NAME); extractor.process(testUri); assertEquals(Collections.EMPTY_SET, testUri.getOutLinks()); @@ -125,17 +125,17 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { public void testPriority() throws Exception { CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); - // 37, 24 are not in url_stream_map; 35 appears before 34 in there, but - // with this list we should get 34 + // 37, 24 are not in url_stream_map; 35 appears before 34 in there, but + // with this list we should get 34 extractor().setItagPriority(Arrays.asList("37", "24", "34", "35")); extractor().setExtractLimit(1); extractor.process(testUri); - Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_SUBSET); + Set expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_SUBSET); assertEquals(expected, testUri.getOutLinks()); } - + public void testAlternatePage() throws Exception { CrawlURI testUri = createTestUri("http://www.youtube.com/watch?v=OyJ3CafAM1Q","ExtractorYoutubeFormatStream2.txt"); @@ -146,44 +146,44 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { } public void testDefaultItag() throws URIException, UnsupportedEncodingException, IOException, InterruptedException { CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME); - + extractor().setExtractLimit(1); - + assertEquals(Collections.EMPTY_LIST, extractor().getItagPriority()); - + extractor.process(testUri); Set expected = makeLinkSet(testUri, EXPECTED_SINGLE_DEFAULT_OUTLINK); assertEquals(expected, testUri.getOutLinks()); } - private Set makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException { - HashSet linkSet = new HashSet(); + private Set makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException { + HashSet linkSet = new HashSet(); for (String urlStr : urlStrs) { - linkSet.add(new Link(sourceUri.getUURI(), + linkSet.add(new Link(sourceUri.getUURI(), UURIFactory.getInstance(urlStr), HTMLLinkContext.EMBED_MISC, Hop.EMBED) - ); + ); } - return linkSet; - } + return linkSet; + } - private ExtractorYoutubeFormatStream extractor() { - return (ExtractorYoutubeFormatStream)extractor; - } - - private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException, - UnsupportedEncodingException, IOException { - UURI testUuri = UURIFactory.getInstance(urlStr); + private ExtractorYoutubeFormatStream extractor() { + return (ExtractorYoutubeFormatStream)extractor; + } + + private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException, + UnsupportedEncodingException, IOException { + UURI testUuri = UURIFactory.getInstance(urlStr); CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC); InputStream is = ExtractorYoutubeFormatStreamTest.class.getClassLoader().getResourceAsStream(resourceFileName); - BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); - StringBuilder content = new StringBuilder(); - String line = ""; - while ((line = reader.readLine()) != null) { - content.append(line); - } + BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + StringBuilder content = new StringBuilder(); + String line = ""; + while ((line = reader.readLine()) != null) { + content.append(line); + } Recorder recorder = createRecorder(content.toString(), "UTF-8"); IOUtils.closeQuietly(is); @@ -191,22 +191,22 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase { testUri.setFetchStatus(200); testUri.setRecorder(recorder); testUri.setContentSize(content.length()); - return testUri; - } + return testUri; + } } class UnitTestUriLoggerModule implements UriErrorLoggerModule { final private static Logger LOGGER = - Logger.getLogger(UnitTestUriLoggerModule.class.getName()); + Logger.getLogger(UnitTestUriLoggerModule.class.getName()); public void info(String info) { - LOGGER.log(Level.INFO, info); - System.out.println("INFO - "+info); + LOGGER.log(Level.INFO, info); + System.out.println("INFO - "+info); } public void fine(String info) { - System.out.println("Fine - "+info); - LOGGER.log(Level.FINE, info); - + System.out.println("Fine - "+info); + LOGGER.log(Level.FINE, info); + } public void logUriError(URIException e, UURI u, CharSequence l) { LOGGER.log(Level.INFO, u.toString(), e); @@ -217,6 +217,6 @@ class UnitTestUriLoggerModule implements UriErrorLoggerModule { return LOGGER; } - - + + } \ No newline at end of file