mirror of
https://github.com/internetarchive/heritrix3.git
synced 2026-09-26 07:35:39 +00:00
make code formatting consistent with main heritrix proj by changing spaces to tabs; also remove unused imports from HBase.java, no other substantive changes
This commit is contained in:
Noah Levitt
+141
-141
@@ -28,180 +28,180 @@ import org.json.JSONObject;
|
||||
*
|
||||
*/
|
||||
public class ExtractorYoutubeFormatStream extends Extractor {
|
||||
private static Logger logger =
|
||||
private static Logger logger =
|
||||
Logger.getLogger(ExtractorYoutubeFormatStream.class.getName());
|
||||
|
||||
{
|
||||
setExtractLimit(1);
|
||||
setExtractLimit(1);
|
||||
}
|
||||
public Integer getExtractLimit(){
|
||||
return (Integer) kp.get("extractLimit");
|
||||
return (Integer) kp.get("extractLimit");
|
||||
}
|
||||
/**
|
||||
* Maximum number of video urls to extract. A value of 0 means extract all
|
||||
* discovered video urls. Default is 1.
|
||||
*/
|
||||
public void setExtractLimit(Integer extractLimit){
|
||||
kp.put("extractLimit", extractLimit);
|
||||
kp.put("extractLimit", extractLimit);
|
||||
}
|
||||
|
||||
{
|
||||
setItagPriority(new ArrayList<String>());
|
||||
setItagPriority(new ArrayList<String>());
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
public List<String> getItagPriority() {
|
||||
return (List<String>) kp.get("itagPriority");
|
||||
return (List<String>) kp.get("itagPriority");
|
||||
}
|
||||
|
||||
/**
|
||||
* Itag priority list. Youtube itag parameter specifies the video and audio
|
||||
* format and quality. The default is an empty list, which tells the
|
||||
* extractor to extract up to extractLimit video urls. When the
|
||||
* list is not empty, only video urls with itag values in the list are
|
||||
* extracted.
|
||||
*
|
||||
* @see <a
|
||||
* href="http://en.wikipedia.org/wiki/YouTube">http://en.wikipedia.org/wiki/YouTube</a>
|
||||
*/
|
||||
|
||||
/**
|
||||
* Itag priority list. Youtube itag parameter specifies the video and audio
|
||||
* format and quality. The default is an empty list, which tells the
|
||||
* extractor to extract up to extractLimit video urls. When the
|
||||
* list is not empty, only video urls with itag values in the list are
|
||||
* extracted.
|
||||
*
|
||||
* @see <a
|
||||
* href="http://en.wikipedia.org/wiki/YouTube">http://en.wikipedia.org/wiki/YouTube</a>
|
||||
*/
|
||||
public void setItagPriority(List<String> itagPriority) {
|
||||
kp.put("itagPriority", itagPriority);
|
||||
kp.put("itagPriority", itagPriority);
|
||||
}
|
||||
|
||||
@Override
|
||||
@Override
|
||||
protected boolean shouldProcess(CrawlURI uri) {
|
||||
return uri.getContentLength() > 0
|
||||
&& uri.getFetchStatus() == 200
|
||||
&& TextUtils.matches("^https?://([^.]+\\.)?youtube\\.com/watch.*$",
|
||||
uri.getUURI().toCustomString());
|
||||
return uri.getContentLength() > 0
|
||||
&& uri.getFetchStatus() == 200
|
||||
&& TextUtils.matches("^https?://([^.]+\\.)?youtube\\.com/watch.*$",
|
||||
uri.getUURI().toCustomString());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void extract(CrawlURI uri) {
|
||||
ReplayCharSequence cs;
|
||||
try {
|
||||
cs = uri.getRecorder().getContentReplayCharSequence();
|
||||
} catch (IOException e) {
|
||||
uri.getNonFatalFailures().add(e);
|
||||
logger.log(Level.WARNING, "Failed get of replay char sequence in "
|
||||
+ Thread.currentThread().getName(), e);
|
||||
return;
|
||||
}
|
||||
@Override
|
||||
protected void extract(CrawlURI uri) {
|
||||
ReplayCharSequence cs;
|
||||
try {
|
||||
cs = uri.getRecorder().getContentReplayCharSequence();
|
||||
} catch (IOException e) {
|
||||
uri.getNonFatalFailures().add(e);
|
||||
logger.log(Level.WARNING, "Failed get of replay char sequence in "
|
||||
+ Thread.currentThread().getName(), e);
|
||||
return;
|
||||
}
|
||||
|
||||
Matcher matcher = TextUtils.getMatcher(
|
||||
"(?is)ytplayer\\.config = (\\{.*?\\})(;|</script>|$)", cs);
|
||||
if (matcher.find()) {
|
||||
String jsonStr = matcher.group(1);
|
||||
Matcher matcher = TextUtils.getMatcher(
|
||||
"(?is)ytplayer\\.config = (\\{.*?\\})(;|</script>|$)", cs);
|
||||
if (matcher.find()) {
|
||||
String jsonStr = matcher.group(1);
|
||||
|
||||
// logger.fine("Just Extracted: "+jsonStr);
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonStr);
|
||||
if (json.has("args")) {
|
||||
JSONObject args = json.getJSONObject("args");
|
||||
if (args.has("url_encoded_fmt_stream_map")) {
|
||||
String streamMap = args.getString("url_encoded_fmt_stream_map");
|
||||
// logger.fine("Just Extracted: "+jsonStr);
|
||||
try {
|
||||
JSONObject json = new JSONObject(jsonStr);
|
||||
if (json.has("args")) {
|
||||
JSONObject args = json.getJSONObject("args");
|
||||
if (args.has("url_encoded_fmt_stream_map")) {
|
||||
String streamMap = args.getString("url_encoded_fmt_stream_map");
|
||||
|
||||
// logger.info("Just Extracted: "+stream_map);
|
||||
LinkedHashMap<String, String> parsedVideoMap = parseStreamMap(streamMap);
|
||||
addPreferredOutlinks(uri, parsedVideoMap);
|
||||
}
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
logger.log(Level.WARNING,
|
||||
"Error parsing JSON object - Skipping: " + jsonStr, e);
|
||||
}
|
||||
}
|
||||
TextUtils.recycleMatcher(matcher);
|
||||
}
|
||||
// logger.info("Just Extracted: "+stream_map);
|
||||
LinkedHashMap<String, String> parsedVideoMap = parseStreamMap(streamMap);
|
||||
addPreferredOutlinks(uri, parsedVideoMap);
|
||||
}
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
logger.log(Level.WARNING,
|
||||
"Error parsing JSON object - Skipping: " + jsonStr, e);
|
||||
}
|
||||
}
|
||||
TextUtils.recycleMatcher(matcher);
|
||||
}
|
||||
|
||||
// 34 and 35 are most common medium quality flvs, others are in arbitrary order
|
||||
private static final List<String> DEFAULT_ITAG_PRIORITY = Arrays.asList(
|
||||
"35", "34", "5", "6", "13", "17", "18", "22", "36", "37", "38",
|
||||
"43", "44", "45", "46", "82", "83", "84", "85", "100", "101",
|
||||
"102", "120");
|
||||
private static final Set<String> KNOWN_ITAGS = new HashSet<String>(DEFAULT_ITAG_PRIORITY);
|
||||
|
||||
// Add videos as outlinks by priority list
|
||||
private void addPreferredOutlinks(CrawlURI uri,
|
||||
LinkedHashMap<String, String> parsedVideoMap) {
|
||||
List<String> itagPriority;
|
||||
if (getItagPriority() != null && !getItagPriority().isEmpty()) {
|
||||
itagPriority = getItagPriority();
|
||||
} else {
|
||||
itagPriority = DEFAULT_ITAG_PRIORITY;
|
||||
}
|
||||
|
||||
int extractionCount = 0;
|
||||
for (String itag : itagPriority) {
|
||||
if (parsedVideoMap.containsKey(itag)
|
||||
&& (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) {
|
||||
logger.fine("adding video: " + parsedVideoMap.get(itag));
|
||||
addOutlink(uri, parsedVideoMap.get(itag),
|
||||
org.archive.modules.extractor.LinkContext.EMBED_MISC,
|
||||
org.archive.modules.extractor.Hop.EMBED);
|
||||
extractionCount++;
|
||||
}
|
||||
}
|
||||
// 34 and 35 are most common medium quality flvs, others are in arbitrary order
|
||||
private static final List<String> DEFAULT_ITAG_PRIORITY = Arrays.asList(
|
||||
"35", "34", "5", "6", "13", "17", "18", "22", "36", "37", "38",
|
||||
"43", "44", "45", "46", "82", "83", "84", "85", "100", "101",
|
||||
"102", "120");
|
||||
private static final Set<String> KNOWN_ITAGS = new HashSet<String>(DEFAULT_ITAG_PRIORITY);
|
||||
|
||||
// if itagPriority not specified, make sure we consider all discovered
|
||||
// video urls
|
||||
if (getItagPriority() == null || getItagPriority().isEmpty()) {
|
||||
Iterator<String> itagKeyIter = parsedVideoMap.keySet().iterator();
|
||||
while (itagKeyIter.hasNext() && (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) {
|
||||
String itag = itagKeyIter.next();
|
||||
if (!KNOWN_ITAGS.contains(itag)) {
|
||||
logger.warning("adding video (with unknown itag " + itag
|
||||
+ "): " + parsedVideoMap.get(itag));
|
||||
addOutlink(uri, parsedVideoMap.get(itag),
|
||||
org.archive.modules.extractor.LinkContext.EMBED_MISC,
|
||||
org.archive.modules.extractor.Hop.EMBED);
|
||||
extractionCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private LinkedHashMap<String, String> parseStreamMap(String streamMap) {
|
||||
String[] rawVideoList = streamMap.split(",");
|
||||
LinkedHashMap<String, String> parsedVideoMap = new LinkedHashMap<String, String>();
|
||||
// Add videos as outlinks by priority list
|
||||
private void addPreferredOutlinks(CrawlURI uri,
|
||||
LinkedHashMap<String, String> parsedVideoMap) {
|
||||
List<String> itagPriority;
|
||||
if (getItagPriority() != null && !getItagPriority().isEmpty()) {
|
||||
itagPriority = getItagPriority();
|
||||
} else {
|
||||
itagPriority = DEFAULT_ITAG_PRIORITY;
|
||||
}
|
||||
|
||||
// Parse Video Map into itag,url pair
|
||||
for (int i = 0; i < rawVideoList.length; i++) {
|
||||
String[] videoParams = rawVideoList[i].split("\\u0026");
|
||||
String videoURLParam, itagParam, sigParam;
|
||||
videoURLParam = itagParam = sigParam = "";
|
||||
int extractionCount = 0;
|
||||
for (String itag : itagPriority) {
|
||||
if (parsedVideoMap.containsKey(itag)
|
||||
&& (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) {
|
||||
logger.fine("adding video: " + parsedVideoMap.get(itag));
|
||||
addOutlink(uri, parsedVideoMap.get(itag),
|
||||
org.archive.modules.extractor.LinkContext.EMBED_MISC,
|
||||
org.archive.modules.extractor.Hop.EMBED);
|
||||
extractionCount++;
|
||||
}
|
||||
}
|
||||
|
||||
for (String param : videoParams) {
|
||||
// if itagPriority not specified, make sure we consider all discovered
|
||||
// video urls
|
||||
if (getItagPriority() == null || getItagPriority().isEmpty()) {
|
||||
Iterator<String> itagKeyIter = parsedVideoMap.keySet().iterator();
|
||||
while (itagKeyIter.hasNext() && (getExtractLimit() <= 0 || extractionCount < getExtractLimit())) {
|
||||
String itag = itagKeyIter.next();
|
||||
if (!KNOWN_ITAGS.contains(itag)) {
|
||||
logger.warning("adding video (with unknown itag " + itag
|
||||
+ "): " + parsedVideoMap.get(itag));
|
||||
addOutlink(uri, parsedVideoMap.get(itag),
|
||||
org.archive.modules.extractor.LinkContext.EMBED_MISC,
|
||||
org.archive.modules.extractor.Hop.EMBED);
|
||||
extractionCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
String[] keyValuePair = param.split("=");
|
||||
if (keyValuePair.length != 2) {
|
||||
logger.warning("Invalid Video Parameter: " + param);
|
||||
continue;
|
||||
}
|
||||
private LinkedHashMap<String, String> parseStreamMap(String streamMap) {
|
||||
String[] rawVideoList = streamMap.split(",");
|
||||
LinkedHashMap<String, String> parsedVideoMap = new LinkedHashMap<String, String>();
|
||||
|
||||
if (keyValuePair[0].equals("url")) {
|
||||
videoURLParam = keyValuePair[1];
|
||||
}
|
||||
if (keyValuePair[0].equals("itag")) {
|
||||
itagParam = keyValuePair[1];
|
||||
}
|
||||
if (keyValuePair[0].equals("sig")) {
|
||||
sigParam = keyValuePair[1];
|
||||
}
|
||||
}
|
||||
// Parse Video Map into itag,url pair
|
||||
for (int i = 0; i < rawVideoList.length; i++) {
|
||||
String[] videoParams = rawVideoList[i].split("\\u0026");
|
||||
String videoURLParam, itagParam, sigParam;
|
||||
videoURLParam = itagParam = sigParam = "";
|
||||
|
||||
if (videoURLParam.length() > 0 && itagParam.length() > 0
|
||||
&& sigParam.length() > 0) {
|
||||
try {
|
||||
String fixupURL = URLDecoder.decode(videoURLParam
|
||||
+ "%26signature=" + sigParam, "UTF-8");
|
||||
parsedVideoMap.put(itagParam, fixupURL);
|
||||
} catch (java.io.UnsupportedEncodingException e) {
|
||||
logger.warning("Error decoding youtube video URL: "
|
||||
+ videoURLParam + "%26signature=" + sigParam);
|
||||
}
|
||||
}
|
||||
}
|
||||
return parsedVideoMap;
|
||||
}
|
||||
for (String param : videoParams) {
|
||||
|
||||
String[] keyValuePair = param.split("=");
|
||||
if (keyValuePair.length != 2) {
|
||||
logger.warning("Invalid Video Parameter: " + param);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (keyValuePair[0].equals("url")) {
|
||||
videoURLParam = keyValuePair[1];
|
||||
}
|
||||
if (keyValuePair[0].equals("itag")) {
|
||||
itagParam = keyValuePair[1];
|
||||
}
|
||||
if (keyValuePair[0].equals("sig")) {
|
||||
sigParam = keyValuePair[1];
|
||||
}
|
||||
}
|
||||
|
||||
if (videoURLParam.length() > 0 && itagParam.length() > 0
|
||||
&& sigParam.length() > 0) {
|
||||
try {
|
||||
String fixupURL = URLDecoder.decode(videoURLParam
|
||||
+ "%26signature=" + sigParam, "UTF-8");
|
||||
parsedVideoMap.put(itagParam, fixupURL);
|
||||
} catch (java.io.UnsupportedEncodingException e) {
|
||||
logger.warning("Error decoding youtube video URL: "
|
||||
+ videoURLParam + "%26signature=" + sigParam);
|
||||
}
|
||||
}
|
||||
}
|
||||
return parsedVideoMap;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -18,15 +18,9 @@
|
||||
*/
|
||||
package org.archive.modules.recrawl.hbase;
|
||||
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_FETCH_HISTORY;
|
||||
import static org.archive.modules.recrawl.RecrawlAttributeConstants.A_WRITE_TAG;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.logging.LogManager;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
@@ -35,9 +29,6 @@ import org.apache.hadoop.hbase.MasterNotRunningException;
|
||||
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
|
||||
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
||||
import org.apache.hadoop.hbase.client.HConnectionManager;
|
||||
import org.archive.modules.CrawlURI;
|
||||
import org.archive.modules.recrawl.FetchHistoryProcessor;
|
||||
import org.archive.net.UURIFactory;
|
||||
import org.springframework.context.Lifecycle;
|
||||
|
||||
/**
|
||||
@@ -47,71 +38,71 @@ import org.springframework.context.Lifecycle;
|
||||
*/
|
||||
public class HBase implements Lifecycle {
|
||||
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(HBase.class.getName());
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(HBase.class.getName());
|
||||
|
||||
protected Configuration conf = null;
|
||||
protected Configuration conf = null;
|
||||
|
||||
private Map<String,String> properties;
|
||||
|
||||
public Map<String,String> getProperties() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
public void setProperties(Map<String,String> properties) {
|
||||
this.properties = properties;
|
||||
private Map<String,String> properties;
|
||||
|
||||
if (conf == null) {
|
||||
conf = HBaseConfiguration.create();
|
||||
}
|
||||
for (Entry<String, String> entry: getProperties().entrySet()) {
|
||||
conf.set(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
public Map<String,String> getProperties() {
|
||||
return properties;
|
||||
}
|
||||
|
||||
public synchronized Configuration configuration() {
|
||||
if (conf == null) {
|
||||
conf = HBaseConfiguration.create();
|
||||
}
|
||||
public void setProperties(Map<String,String> properties) {
|
||||
this.properties = properties;
|
||||
|
||||
return conf;
|
||||
}
|
||||
if (conf == null) {
|
||||
conf = HBaseConfiguration.create();
|
||||
}
|
||||
for (Entry<String, String> entry: getProperties().entrySet()) {
|
||||
conf.set(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
protected transient HBaseAdmin admin;
|
||||
public synchronized Configuration configuration() {
|
||||
if (conf == null) {
|
||||
conf = HBaseConfiguration.create();
|
||||
}
|
||||
|
||||
public synchronized HBaseAdmin admin() throws MasterNotRunningException, ZooKeeperConnectionException {
|
||||
if (admin == null) {
|
||||
admin = new HBaseAdmin(configuration());
|
||||
}
|
||||
return conf;
|
||||
}
|
||||
|
||||
return admin;
|
||||
}
|
||||
protected transient HBaseAdmin admin;
|
||||
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
isRunning = false;
|
||||
if (admin != null) {
|
||||
try {
|
||||
admin.close();
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem closing HBaseAdmin " + admin + " - " + e);
|
||||
}
|
||||
|
||||
admin = null;
|
||||
}
|
||||
if (conf != null) {
|
||||
HConnectionManager.deleteConnection(conf, true);
|
||||
}
|
||||
}
|
||||
public synchronized HBaseAdmin admin() throws MasterNotRunningException, ZooKeeperConnectionException {
|
||||
if (admin == null) {
|
||||
admin = new HBaseAdmin(configuration());
|
||||
}
|
||||
|
||||
protected transient boolean isRunning = false;
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return isRunning;
|
||||
}
|
||||
return admin;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
isRunning = true;
|
||||
}
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
isRunning = false;
|
||||
if (admin != null) {
|
||||
try {
|
||||
admin.close();
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem closing HBaseAdmin " + admin + " - " + e);
|
||||
}
|
||||
|
||||
admin = null;
|
||||
}
|
||||
if (conf != null) {
|
||||
HConnectionManager.deleteConnection(conf, true);
|
||||
}
|
||||
}
|
||||
|
||||
protected transient boolean isRunning = false;
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return isRunning;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
isRunning = true;
|
||||
}
|
||||
}
|
||||
|
||||
+204
-204
@@ -43,159 +43,159 @@ public class HBaseContentDigestHistory extends AbstractContentDigestHistory impl
|
||||
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(HBaseContentDigestHistory.class.getName());
|
||||
|
||||
protected static final byte[] COLUMN_FAMILY = Bytes.toBytes("f");
|
||||
protected static final byte[] COLUMN = Bytes.toBytes("c");
|
||||
|
||||
protected static final BiMap<String,String> JSON_KEYS_MAP = HashBiMap.create();
|
||||
static {
|
||||
JSON_KEYS_MAP.put(A_CONTENT_DIGEST_COUNT, "c");
|
||||
JSON_KEYS_MAP.put(A_ORIGINAL_URL, "u");
|
||||
JSON_KEYS_MAP.put(A_WARC_RECORD_ID, "i");
|
||||
JSON_KEYS_MAP.put(A_WARC_FILENAME, "f");
|
||||
JSON_KEYS_MAP.put(A_WARC_FILE_OFFSET, "o");
|
||||
JSON_KEYS_MAP.put(A_ORIGINAL_DATE, "d");
|
||||
}
|
||||
|
||||
protected HBaseTable table;
|
||||
public void setTable(HBaseTable table) {
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
protected boolean addColumnFamily = false;
|
||||
public boolean getAddColumnFamily() {
|
||||
return addColumnFamily;
|
||||
}
|
||||
/**
|
||||
* Add the expected column family
|
||||
* {@link HBasePersistProcessor#COLUMN_FAMILY} to the HBase table if the
|
||||
* table doesn't already have it.
|
||||
*/
|
||||
public void setAddColumnFamily(boolean addColumnFamily) {
|
||||
this.addColumnFamily = addColumnFamily;
|
||||
}
|
||||
protected static final byte[] COLUMN_FAMILY = Bytes.toBytes("f");
|
||||
protected static final byte[] COLUMN = Bytes.toBytes("c");
|
||||
|
||||
protected int retryIntervalMs = 10*1000;
|
||||
public int getRetryIntervalMs() {
|
||||
return retryIntervalMs;
|
||||
}
|
||||
public void setRetryIntervalMs(int retryIntervalMs) {
|
||||
this.retryIntervalMs = retryIntervalMs;
|
||||
}
|
||||
protected static final BiMap<String,String> JSON_KEYS_MAP = HashBiMap.create();
|
||||
static {
|
||||
JSON_KEYS_MAP.put(A_CONTENT_DIGEST_COUNT, "c");
|
||||
JSON_KEYS_MAP.put(A_ORIGINAL_URL, "u");
|
||||
JSON_KEYS_MAP.put(A_WARC_RECORD_ID, "i");
|
||||
JSON_KEYS_MAP.put(A_WARC_FILENAME, "f");
|
||||
JSON_KEYS_MAP.put(A_WARC_FILE_OFFSET, "o");
|
||||
JSON_KEYS_MAP.put(A_ORIGINAL_DATE, "d");
|
||||
}
|
||||
|
||||
protected int maxTries = 1;
|
||||
public int getMaxTries() {
|
||||
return maxTries;
|
||||
}
|
||||
public void setMaxTries(int maxTries) {
|
||||
this.maxTries = maxTries;
|
||||
}
|
||||
protected HBaseTable table;
|
||||
public void setTable(HBaseTable table) {
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
protected synchronized void addColumnFamily() {
|
||||
try {
|
||||
HTableDescriptor oldDesc = table.getHtableDescriptor();
|
||||
if (oldDesc.getFamily(COLUMN_FAMILY) == null) {
|
||||
HTableDescriptor newDesc = new HTableDescriptor(oldDesc);
|
||||
newDesc.addFamily(new HColumnDescriptor(COLUMN_FAMILY));
|
||||
logger.info("table does not yet have expected column family, modifying descriptor to " + newDesc);
|
||||
HBaseAdmin hbaseAdmin = table.getHbase().admin();
|
||||
hbaseAdmin.disableTable(table.getName());
|
||||
hbaseAdmin.modifyTable(Bytes.toBytes(table.getName()), newDesc);
|
||||
hbaseAdmin.enableTable(table.getName());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem adding column family: " + e);
|
||||
}
|
||||
}
|
||||
protected boolean addColumnFamily = false;
|
||||
public boolean getAddColumnFamily() {
|
||||
return addColumnFamily;
|
||||
}
|
||||
/**
|
||||
* Add the expected column family
|
||||
* {@link HBasePersistProcessor#COLUMN_FAMILY} to the HBase table if the
|
||||
* table doesn't already have it.
|
||||
*/
|
||||
public void setAddColumnFamily(boolean addColumnFamily) {
|
||||
this.addColumnFamily = addColumnFamily;
|
||||
}
|
||||
|
||||
private boolean isRunning;
|
||||
@Override
|
||||
public void start() {
|
||||
// add column family here to avoid disabling table while another
|
||||
// ToeThread is trying to use it
|
||||
if (getAddColumnFamily()) {
|
||||
addColumnFamily();
|
||||
}
|
||||
this.isRunning = true;
|
||||
}
|
||||
@Override
|
||||
public void stop() {
|
||||
this.isRunning = false;
|
||||
}
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return isRunning;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void load(CrawlURI curi) {
|
||||
// make this call in all cases so that the value is initialized and
|
||||
// WARCWriterProcessor knows it should put the info in there
|
||||
HashMap<String, Object> contentDigestHistory = curi.getContentDigestHistory();
|
||||
protected int retryIntervalMs = 10*1000;
|
||||
public int getRetryIntervalMs() {
|
||||
return retryIntervalMs;
|
||||
}
|
||||
public void setRetryIntervalMs(int retryIntervalMs) {
|
||||
this.retryIntervalMs = retryIntervalMs;
|
||||
}
|
||||
|
||||
byte[] key = Bytes.toBytes(persistKeyFor(curi));
|
||||
Result hbaseResult = tryHbaseGet(curi, new Get(key));
|
||||
protected int maxTries = 1;
|
||||
public int getMaxTries() {
|
||||
return maxTries;
|
||||
}
|
||||
public void setMaxTries(int maxTries) {
|
||||
this.maxTries = maxTries;
|
||||
}
|
||||
|
||||
if (hbaseResult != null) {
|
||||
Map<String, Object> loadedHistory = parseHbaseResult(curi, hbaseResult);
|
||||
|
||||
if (loadedHistory != null) {
|
||||
if (logger.isLoggable(Level.FINER)) {
|
||||
logger.finer("loaded history by digest " + persistKeyFor(curi)
|
||||
+ " for uri " + curi + " - " + loadedHistory);
|
||||
}
|
||||
contentDigestHistory.putAll(loadedHistory);
|
||||
}
|
||||
}
|
||||
}
|
||||
protected synchronized void addColumnFamily() {
|
||||
try {
|
||||
HTableDescriptor oldDesc = table.getHtableDescriptor();
|
||||
if (oldDesc.getFamily(COLUMN_FAMILY) == null) {
|
||||
HTableDescriptor newDesc = new HTableDescriptor(oldDesc);
|
||||
newDesc.addFamily(new HColumnDescriptor(COLUMN_FAMILY));
|
||||
logger.info("table does not yet have expected column family, modifying descriptor to " + newDesc);
|
||||
HBaseAdmin hbaseAdmin = table.getHbase().admin();
|
||||
hbaseAdmin.disableTable(table.getName());
|
||||
hbaseAdmin.modifyTable(Bytes.toBytes(table.getName()), newDesc);
|
||||
hbaseAdmin.enableTable(table.getName());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem adding column family: " + e);
|
||||
}
|
||||
}
|
||||
|
||||
protected Result tryHbaseGet(CrawlURI curi, Get hbaseGet) {
|
||||
try {
|
||||
return table.get(hbaseGet);
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem retrieving persist data from hbase, proceeding without, for digest " + persistKeyFor(curi) + " uri " + curi + " - " + e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
private boolean isRunning;
|
||||
@Override
|
||||
public void start() {
|
||||
// add column family here to avoid disabling table while another
|
||||
// ToeThread is trying to use it
|
||||
if (getAddColumnFamily()) {
|
||||
addColumnFamily();
|
||||
}
|
||||
this.isRunning = true;
|
||||
}
|
||||
@Override
|
||||
public void stop() {
|
||||
this.isRunning = false;
|
||||
}
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return isRunning;
|
||||
}
|
||||
|
||||
protected Map<String, Object> parseHbaseResult(CrawlURI curi, Result hbaseResult) {
|
||||
HashMap<String, Object> loadedHistory = null;
|
||||
// no data for uri is indicated by empty Result
|
||||
if (!hbaseResult.isEmpty()) {
|
||||
byte[] jsonBytes = hbaseResult.getValue(COLUMN_FAMILY, COLUMN);
|
||||
if (jsonBytes != null) {
|
||||
JSONObject json = null;
|
||||
try {
|
||||
json = new JSONObject(Bytes.toString(jsonBytes));
|
||||
loadedHistory = new HashMap<String,Object>();
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<String> keyIter = json.keys();
|
||||
while (keyIter.hasNext()) {
|
||||
String jsonKey = keyIter.next();
|
||||
Object jsonValue = json.get(jsonKey);
|
||||
String historyMapKey = JSON_KEYS_MAP.inverse().get(jsonKey);
|
||||
if (historyMapKey == null) {
|
||||
logger.warning("unknown key \"" + jsonKey + "\" found in hbase json for digest " + persistKeyFor(curi));
|
||||
historyMapKey = jsonKey;
|
||||
}
|
||||
loadedHistory.put(historyMapKey, jsonValue);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
logger.warning("problem parsing json for digest " + persistKeyFor(curi) + " uri " + curi + " - " + e);
|
||||
}
|
||||
} else {
|
||||
// shouldn't happen? result.isEmpty() is normal case
|
||||
logger.fine("[jsonBytes==null] no persist data for digest " + persistKeyFor(curi) + " uri " + curi);
|
||||
}
|
||||
} else {
|
||||
logger.finest("[result.isEmpty()] no persist data for digest " + persistKeyFor(curi) + " uri " + curi);
|
||||
}
|
||||
@Override
|
||||
public void load(CrawlURI curi) {
|
||||
// make this call in all cases so that the value is initialized and
|
||||
// WARCWriterProcessor knows it should put the info in there
|
||||
HashMap<String, Object> contentDigestHistory = curi.getContentDigestHistory();
|
||||
|
||||
return loadedHistory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void store(CrawlURI curi) {
|
||||
byte[] key = Bytes.toBytes(persistKeyFor(curi));
|
||||
Result hbaseResult = tryHbaseGet(curi, new Get(key));
|
||||
|
||||
if (hbaseResult != null) {
|
||||
Map<String, Object> loadedHistory = parseHbaseResult(curi, hbaseResult);
|
||||
|
||||
if (loadedHistory != null) {
|
||||
if (logger.isLoggable(Level.FINER)) {
|
||||
logger.finer("loaded history by digest " + persistKeyFor(curi)
|
||||
+ " for uri " + curi + " - " + loadedHistory);
|
||||
}
|
||||
contentDigestHistory.putAll(loadedHistory);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected Result tryHbaseGet(CrawlURI curi, Get hbaseGet) {
|
||||
try {
|
||||
return table.get(hbaseGet);
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem retrieving persist data from hbase, proceeding without, for digest " + persistKeyFor(curi) + " uri " + curi + " - " + e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
protected Map<String, Object> parseHbaseResult(CrawlURI curi, Result hbaseResult) {
|
||||
HashMap<String, Object> loadedHistory = null;
|
||||
// no data for uri is indicated by empty Result
|
||||
if (!hbaseResult.isEmpty()) {
|
||||
byte[] jsonBytes = hbaseResult.getValue(COLUMN_FAMILY, COLUMN);
|
||||
if (jsonBytes != null) {
|
||||
JSONObject json = null;
|
||||
try {
|
||||
json = new JSONObject(Bytes.toString(jsonBytes));
|
||||
loadedHistory = new HashMap<String,Object>();
|
||||
@SuppressWarnings("unchecked")
|
||||
Iterator<String> keyIter = json.keys();
|
||||
while (keyIter.hasNext()) {
|
||||
String jsonKey = keyIter.next();
|
||||
Object jsonValue = json.get(jsonKey);
|
||||
String historyMapKey = JSON_KEYS_MAP.inverse().get(jsonKey);
|
||||
if (historyMapKey == null) {
|
||||
logger.warning("unknown key \"" + jsonKey + "\" found in hbase json for digest " + persistKeyFor(curi));
|
||||
historyMapKey = jsonKey;
|
||||
}
|
||||
loadedHistory.put(historyMapKey, jsonValue);
|
||||
}
|
||||
} catch (JSONException e) {
|
||||
logger.warning("problem parsing json for digest " + persistKeyFor(curi) + " uri " + curi + " - " + e);
|
||||
}
|
||||
} else {
|
||||
// shouldn't happen? result.isEmpty() is normal case
|
||||
logger.fine("[jsonBytes==null] no persist data for digest " + persistKeyFor(curi) + " uri " + curi);
|
||||
}
|
||||
} else {
|
||||
logger.finest("[result.isEmpty()] no persist data for digest " + persistKeyFor(curi) + " uri " + curi);
|
||||
}
|
||||
|
||||
return loadedHistory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void store(CrawlURI curi) {
|
||||
if (!curi.hasContentDigestHistory()
|
||||
|| curi.getContentDigestHistory().isEmpty()) {
|
||||
logger.warning("not saving empty content digest history (do you "
|
||||
@@ -208,69 +208,69 @@ public class HBaseContentDigestHistory extends AbstractContentDigestHistory impl
|
||||
+ " for uri " + curi + " - "
|
||||
+ curi.getContentDigestHistory());
|
||||
}
|
||||
|
||||
Put hbasePut = createHbasePut(curi);
|
||||
tryHbasePut(curi, hbasePut);
|
||||
}
|
||||
|
||||
protected void tryHbasePut(CrawlURI curi, Put p) {
|
||||
int tryCount = 0;
|
||||
do {
|
||||
tryCount++;
|
||||
try {
|
||||
table.put(p);
|
||||
return;
|
||||
} catch (RetriesExhaustedWithDetailsException e) {
|
||||
if (e.getCause(0) instanceof NoSuchColumnFamilyException && getAddColumnFamily()) {
|
||||
addColumnFamily();
|
||||
tryCount--;
|
||||
} else {
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + curi + " - " + e);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + curi + " - " + e);
|
||||
} catch (NullPointerException e) {
|
||||
// HTable.put() throws NullPointerException while connection is lost.
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + curi + " - " + e);
|
||||
}
|
||||
Put hbasePut = createHbasePut(curi);
|
||||
tryHbasePut(curi, hbasePut);
|
||||
}
|
||||
|
||||
if (tryCount > 0 && tryCount < getMaxTries() && isRunning()) {
|
||||
try {
|
||||
Thread.sleep(getRetryIntervalMs());
|
||||
} catch (InterruptedException ex) {
|
||||
logger.warning("thread interrupted. aborting retry for " + curi);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} while (tryCount < getMaxTries() && isRunning());
|
||||
protected void tryHbasePut(CrawlURI curi, Put p) {
|
||||
int tryCount = 0;
|
||||
do {
|
||||
tryCount++;
|
||||
try {
|
||||
table.put(p);
|
||||
return;
|
||||
} catch (RetriesExhaustedWithDetailsException e) {
|
||||
if (e.getCause(0) instanceof NoSuchColumnFamilyException && getAddColumnFamily()) {
|
||||
addColumnFamily();
|
||||
tryCount--;
|
||||
} else {
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + curi + " - " + e);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + curi + " - " + e);
|
||||
} catch (NullPointerException e) {
|
||||
// HTable.put() throws NullPointerException while connection is lost.
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + curi + " - " + e);
|
||||
}
|
||||
|
||||
if (isRunning()) {
|
||||
logger.warning("giving up after " + tryCount + " tries on put for " + curi);
|
||||
}
|
||||
}
|
||||
|
||||
protected Put createHbasePut(CrawlURI curi) {
|
||||
byte[] key = Bytes.toBytes(persistKeyFor(curi));
|
||||
Put hbasePut = new Put(key);
|
||||
try {
|
||||
JSONObject json = new JSONObject();
|
||||
for (Entry<String, Object> entry: curi.getContentDigestHistory().entrySet()) {
|
||||
String jsonKey = JSON_KEYS_MAP.get(entry.getKey());
|
||||
if (jsonKey == null) {
|
||||
logger.warning("unknown key \"" + entry.getKey() + "\" found in content digest history map for " + curi);
|
||||
jsonKey = entry.getKey();
|
||||
}
|
||||
json.put(jsonKey, entry.getValue());
|
||||
}
|
||||
hbasePut.add(COLUMN_FAMILY, COLUMN, Bytes.toBytes(json.toString()));
|
||||
} catch (JSONException e) {
|
||||
// should not happen - all values are either primitive or String.
|
||||
logger.log(Level.SEVERE, "problem creating json object for digest " + persistKeyFor(curi) + " uri " + curi, e);
|
||||
}
|
||||
return hbasePut;
|
||||
}
|
||||
if (tryCount > 0 && tryCount < getMaxTries() && isRunning()) {
|
||||
try {
|
||||
Thread.sleep(getRetryIntervalMs());
|
||||
} catch (InterruptedException ex) {
|
||||
logger.warning("thread interrupted. aborting retry for " + curi);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} while (tryCount < getMaxTries() && isRunning());
|
||||
|
||||
if (isRunning()) {
|
||||
logger.warning("giving up after " + tryCount + " tries on put for " + curi);
|
||||
}
|
||||
}
|
||||
|
||||
protected Put createHbasePut(CrawlURI curi) {
|
||||
byte[] key = Bytes.toBytes(persistKeyFor(curi));
|
||||
Put hbasePut = new Put(key);
|
||||
try {
|
||||
JSONObject json = new JSONObject();
|
||||
for (Entry<String, Object> entry: curi.getContentDigestHistory().entrySet()) {
|
||||
String jsonKey = JSON_KEYS_MAP.get(entry.getKey());
|
||||
if (jsonKey == null) {
|
||||
logger.warning("unknown key \"" + entry.getKey() + "\" found in content digest history map for " + curi);
|
||||
jsonKey = entry.getKey();
|
||||
}
|
||||
json.put(jsonKey, entry.getValue());
|
||||
}
|
||||
hbasePut.add(COLUMN_FAMILY, COLUMN, Bytes.toBytes(json.toString()));
|
||||
} catch (JSONException e) {
|
||||
// should not happen - all values are either primitive or String.
|
||||
logger.log(Level.SEVERE, "problem creating json object for digest " + persistKeyFor(curi) + " uri " + curi, e);
|
||||
}
|
||||
return hbasePut;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+52
-52
@@ -40,60 +40,60 @@ import org.archive.modules.recrawl.RecrawlAttributeConstants;
|
||||
* @contributor kenji
|
||||
*/
|
||||
public class HBasePersistLoadProcessor extends HBasePersistProcessor {
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(HBasePersistLoadProcessor.class.getName());
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(HBasePersistLoadProcessor.class.getName());
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
protected static Map<String, Object>[] getFetchHistory(CrawlURI uri) {
|
||||
Map<String, Object> data = uri.getData();
|
||||
Map<String, Object>[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY);
|
||||
if (history == null) {
|
||||
// only the first element is used by FetchHTTP, WarcWriterProcessor etc.
|
||||
// FetchHistoryProcessor casts history to HashMap[]. So it must be new HashMap[1].
|
||||
history = new HashMap[2];
|
||||
data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history);
|
||||
}
|
||||
return history;
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
protected static Map<String, Object>[] getFetchHistory(CrawlURI uri) {
|
||||
Map<String, Object> data = uri.getData();
|
||||
Map<String, Object>[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY);
|
||||
if (history == null) {
|
||||
// only the first element is used by FetchHTTP, WarcWriterProcessor etc.
|
||||
// FetchHistoryProcessor casts history to HashMap[]. So it must be new HashMap[1].
|
||||
history = new HashMap[2];
|
||||
data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history);
|
||||
}
|
||||
return history;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ProcessResult innerProcessResult(CrawlURI uri) throws InterruptedException {
|
||||
byte[] key = Bytes.toBytes(PersistProcessor.persistKeyFor(uri));
|
||||
Get g = new Get(key);
|
||||
try {
|
||||
Result r = table.get(g);
|
||||
// no data for uri is indicated by empty Result
|
||||
if (r.isEmpty()) {
|
||||
if (logger.isLoggable(Level.FINE)) {
|
||||
logger.fine(uri + ": <no crawlinfo>");
|
||||
}
|
||||
return ProcessResult.PROCEED;
|
||||
}
|
||||
schema.load(r, uri);
|
||||
if (uri.getFetchStatus() < 0) {
|
||||
return ProcessResult.FINISH;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem retrieving persist data from hbase, proceeding without, for " + uri + " - " + e);
|
||||
}
|
||||
return ProcessResult.PROCEED;
|
||||
}
|
||||
@Override
|
||||
protected ProcessResult innerProcessResult(CrawlURI uri) throws InterruptedException {
|
||||
byte[] key = Bytes.toBytes(PersistProcessor.persistKeyFor(uri));
|
||||
Get g = new Get(key);
|
||||
try {
|
||||
Result r = table.get(g);
|
||||
// no data for uri is indicated by empty Result
|
||||
if (r.isEmpty()) {
|
||||
if (logger.isLoggable(Level.FINE)) {
|
||||
logger.fine(uri + ": <no crawlinfo>");
|
||||
}
|
||||
return ProcessResult.PROCEED;
|
||||
}
|
||||
schema.load(r, uri);
|
||||
if (uri.getFetchStatus() < 0) {
|
||||
return ProcessResult.FINISH;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem retrieving persist data from hbase, proceeding without, for " + uri + " - " + e);
|
||||
}
|
||||
return ProcessResult.PROCEED;
|
||||
}
|
||||
|
||||
/**
|
||||
* unused.
|
||||
*/
|
||||
@Override
|
||||
protected void innerProcess(CrawlURI uri) throws InterruptedException {
|
||||
}
|
||||
/**
|
||||
* unused.
|
||||
*/
|
||||
@Override
|
||||
protected void innerProcess(CrawlURI uri) throws InterruptedException {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean shouldProcess(CrawlURI uri) {
|
||||
// TODO: we want deduplicate robots.txt, too.
|
||||
//if (uri.isPrerequisite()) return false;
|
||||
String scheme = uri.getUURI().getScheme();
|
||||
if (!(scheme.equals("http") || scheme.equals("https") || scheme.equals("ftp"))) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@Override
|
||||
protected boolean shouldProcess(CrawlURI uri) {
|
||||
// TODO: we want deduplicate robots.txt, too.
|
||||
//if (uri.isPrerequisite()) return false;
|
||||
String scheme = uri.getUURI().getScheme();
|
||||
if (!(scheme.equals("http") || scheme.equals("https") || scheme.equals("ftp"))) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
+15
-15
@@ -19,21 +19,21 @@ import org.springframework.beans.factory.annotation.Required;
|
||||
*/
|
||||
public abstract class HBasePersistProcessor extends AbstractPersistProcessor {
|
||||
|
||||
protected HBaseTable table;
|
||||
public void setTable(HBaseTable table) {
|
||||
this.table = table;
|
||||
}
|
||||
protected HBaseTable table;
|
||||
public void setTable(HBaseTable table) {
|
||||
this.table = table;
|
||||
}
|
||||
|
||||
protected RecrawlDataSchema schema;
|
||||
public RecrawlDataSchema getSchema() {
|
||||
return schema;
|
||||
}
|
||||
@Required
|
||||
public void setSchema(RecrawlDataSchema schema) {
|
||||
this.schema = schema;
|
||||
}
|
||||
protected RecrawlDataSchema schema;
|
||||
public RecrawlDataSchema getSchema() {
|
||||
return schema;
|
||||
}
|
||||
@Required
|
||||
public void setSchema(RecrawlDataSchema schema) {
|
||||
this.schema = schema;
|
||||
}
|
||||
|
||||
public HBasePersistProcessor() {
|
||||
super();
|
||||
}
|
||||
public HBasePersistProcessor() {
|
||||
super();
|
||||
}
|
||||
}
|
||||
|
||||
+86
-86
@@ -36,98 +36,98 @@ import org.archive.modules.recrawl.RecrawlAttributeConstants;
|
||||
* @contributor kenji
|
||||
*/
|
||||
public class HBasePersistStoreProcessor extends HBasePersistProcessor implements FetchStatusCodes, RecrawlAttributeConstants {
|
||||
private static final Logger logger = Logger.getLogger(HBasePersistStoreProcessor.class.getName());
|
||||
private static final Logger logger = Logger.getLogger(HBasePersistStoreProcessor.class.getName());
|
||||
|
||||
protected boolean addColumnFamily = false;
|
||||
public boolean getAddColumnFamily() {
|
||||
return addColumnFamily;
|
||||
}
|
||||
/**
|
||||
* Add the expected column family
|
||||
* {@link HBasePersistProcessor#COLUMN_FAMILY} to the HBase table if the
|
||||
* table doesn't already have it.
|
||||
*/
|
||||
public void setAddColumnFamily(boolean addColumnFamily) {
|
||||
this.addColumnFamily = addColumnFamily;
|
||||
}
|
||||
protected boolean addColumnFamily = false;
|
||||
public boolean getAddColumnFamily() {
|
||||
return addColumnFamily;
|
||||
}
|
||||
/**
|
||||
* Add the expected column family
|
||||
* {@link HBasePersistProcessor#COLUMN_FAMILY} to the HBase table if the
|
||||
* table doesn't already have it.
|
||||
*/
|
||||
public void setAddColumnFamily(boolean addColumnFamily) {
|
||||
this.addColumnFamily = addColumnFamily;
|
||||
}
|
||||
|
||||
protected int retryIntervalMs = 10*1000;
|
||||
public int getRetryIntervalMs() {
|
||||
return retryIntervalMs;
|
||||
}
|
||||
public void setRetryIntervalMs(int retryIntervalMs) {
|
||||
this.retryIntervalMs = retryIntervalMs;
|
||||
}
|
||||
protected int retryIntervalMs = 10*1000;
|
||||
public int getRetryIntervalMs() {
|
||||
return retryIntervalMs;
|
||||
}
|
||||
public void setRetryIntervalMs(int retryIntervalMs) {
|
||||
this.retryIntervalMs = retryIntervalMs;
|
||||
}
|
||||
|
||||
protected int maxTries = 1;
|
||||
public int getMaxTries() {
|
||||
return maxTries;
|
||||
}
|
||||
public void setMaxTries(int maxTries) {
|
||||
this.maxTries = maxTries;
|
||||
}
|
||||
protected int maxTries = 1;
|
||||
public int getMaxTries() {
|
||||
return maxTries;
|
||||
}
|
||||
public void setMaxTries(int maxTries) {
|
||||
this.maxTries = maxTries;
|
||||
}
|
||||
|
||||
protected synchronized void addColumnFamily() {
|
||||
try {
|
||||
HTableDescriptor oldDesc = table.getHtableDescriptor();
|
||||
byte[] columnFamily = Bytes.toBytes(schema.getColumnFamily());
|
||||
if (oldDesc.getFamily(columnFamily) == null) {
|
||||
HTableDescriptor newDesc = new HTableDescriptor(oldDesc);
|
||||
newDesc.addFamily(new HColumnDescriptor(columnFamily));
|
||||
logger.info("table does not yet have expected column family, modifying descriptor to " + newDesc);
|
||||
HBaseAdmin hbaseAdmin = table.getHbase().admin();
|
||||
hbaseAdmin.disableTable(table.getName());
|
||||
hbaseAdmin.modifyTable(Bytes.toBytes(table.getName()), newDesc);
|
||||
hbaseAdmin.enableTable(table.getName());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem adding column family: " + e);
|
||||
}
|
||||
}
|
||||
protected synchronized void addColumnFamily() {
|
||||
try {
|
||||
HTableDescriptor oldDesc = table.getHtableDescriptor();
|
||||
byte[] columnFamily = Bytes.toBytes(schema.getColumnFamily());
|
||||
if (oldDesc.getFamily(columnFamily) == null) {
|
||||
HTableDescriptor newDesc = new HTableDescriptor(oldDesc);
|
||||
newDesc.addFamily(new HColumnDescriptor(columnFamily));
|
||||
logger.info("table does not yet have expected column family, modifying descriptor to " + newDesc);
|
||||
HBaseAdmin hbaseAdmin = table.getHbase().admin();
|
||||
hbaseAdmin.disableTable(table.getName());
|
||||
hbaseAdmin.modifyTable(Bytes.toBytes(table.getName()), newDesc);
|
||||
hbaseAdmin.enableTable(table.getName());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem adding column family: " + e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void innerProcess(CrawlURI uri) {
|
||||
Put p = schema.createPut(uri);
|
||||
int tryCount = 0;
|
||||
do {
|
||||
tryCount++;
|
||||
try {
|
||||
table.put(p);
|
||||
return;
|
||||
} catch (RetriesExhaustedWithDetailsException e) {
|
||||
if (e.getCause(0) instanceof NoSuchColumnFamilyException && getAddColumnFamily()) {
|
||||
addColumnFamily();
|
||||
tryCount--;
|
||||
} else {
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + uri + " - " + e);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + uri + " - " + e);
|
||||
} catch (NullPointerException e) {
|
||||
// HTable.put() throws NullPointerException while connection is lost.
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + uri + " - " + e);
|
||||
}
|
||||
@Override
|
||||
protected void innerProcess(CrawlURI uri) {
|
||||
Put p = schema.createPut(uri);
|
||||
int tryCount = 0;
|
||||
do {
|
||||
tryCount++;
|
||||
try {
|
||||
table.put(p);
|
||||
return;
|
||||
} catch (RetriesExhaustedWithDetailsException e) {
|
||||
if (e.getCause(0) instanceof NoSuchColumnFamilyException && getAddColumnFamily()) {
|
||||
addColumnFamily();
|
||||
tryCount--;
|
||||
} else {
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + uri + " - " + e);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + uri + " - " + e);
|
||||
} catch (NullPointerException e) {
|
||||
// HTable.put() throws NullPointerException while connection is lost.
|
||||
logger.warning("put failed " + "(try " + tryCount + " of "
|
||||
+ getMaxTries() + ")" + " for " + uri + " - " + e);
|
||||
}
|
||||
|
||||
if (tryCount > 0 && tryCount < getMaxTries() && isRunning()) {
|
||||
try {
|
||||
Thread.sleep(getRetryIntervalMs());
|
||||
} catch (InterruptedException ex) {
|
||||
logger.warning("thread interrupted. aborting retry for " + uri);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} while (tryCount < getMaxTries() && isRunning());
|
||||
if (tryCount > 0 && tryCount < getMaxTries() && isRunning()) {
|
||||
try {
|
||||
Thread.sleep(getRetryIntervalMs());
|
||||
} catch (InterruptedException ex) {
|
||||
logger.warning("thread interrupted. aborting retry for " + uri);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} while (tryCount < getMaxTries() && isRunning());
|
||||
|
||||
if (isRunning()) {
|
||||
logger.warning("giving up after " + tryCount + " tries on put for " + uri);
|
||||
}
|
||||
}
|
||||
if (isRunning()) {
|
||||
logger.warning("giving up after " + tryCount + " tries on put for " + uri);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected boolean shouldProcess(CrawlURI curi) {
|
||||
return super.shouldStore(curi);
|
||||
}
|
||||
@Override
|
||||
protected boolean shouldProcess(CrawlURI curi) {
|
||||
return super.shouldStore(curi);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,116 +38,116 @@ import org.springframework.context.Lifecycle;
|
||||
*/
|
||||
public class HBaseTable implements Lifecycle {
|
||||
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(HBaseTable.class.getName());
|
||||
private static final Logger logger =
|
||||
Logger.getLogger(HBaseTable.class.getName());
|
||||
|
||||
// XXX this default doesn't really belong here if this is supposed to be a
|
||||
// generic hbase table class
|
||||
protected String name = PersistOnlineProcessor.URI_HISTORY_DBNAME;
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
// XXX this default doesn't really belong here if this is supposed to be a
|
||||
// generic hbase table class
|
||||
protected String name = PersistOnlineProcessor.URI_HISTORY_DBNAME;
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
protected HBase hbase = new HBase();
|
||||
public void setHbase(HBase hbase) {
|
||||
this.hbase = hbase;
|
||||
}
|
||||
public HBase getHbase() {
|
||||
return hbase;
|
||||
}
|
||||
protected HBase hbase = new HBase();
|
||||
public void setHbase(HBase hbase) {
|
||||
this.hbase = hbase;
|
||||
}
|
||||
public HBase getHbase() {
|
||||
return hbase;
|
||||
}
|
||||
|
||||
protected boolean create = false;
|
||||
public boolean getCreate() {
|
||||
return create;
|
||||
}
|
||||
/** Create the named table if it doesn't exist. */
|
||||
public void setCreate(boolean create) {
|
||||
this.create = create;
|
||||
}
|
||||
protected boolean create = false;
|
||||
public boolean getCreate() {
|
||||
return create;
|
||||
}
|
||||
/** Create the named table if it doesn't exist. */
|
||||
public void setCreate(boolean create) {
|
||||
this.create = create;
|
||||
}
|
||||
|
||||
protected HTablePool htablePool = null;
|
||||
protected HTablePool htablePool = null;
|
||||
|
||||
public HBaseTable() {
|
||||
}
|
||||
public HBaseTable() {
|
||||
}
|
||||
|
||||
protected synchronized HTablePool htablePool() throws IOException {
|
||||
if (htablePool == null) {
|
||||
// XXX maxSize = number of toe threads?
|
||||
htablePool = new HTablePool(hbase.configuration(),
|
||||
Integer.MAX_VALUE);
|
||||
}
|
||||
protected synchronized HTablePool htablePool() throws IOException {
|
||||
if (htablePool == null) {
|
||||
// XXX maxSize = number of toe threads?
|
||||
htablePool = new HTablePool(hbase.configuration(),
|
||||
Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
return htablePool;
|
||||
}
|
||||
return htablePool;
|
||||
}
|
||||
|
||||
public void put(Put p) throws IOException {
|
||||
HTableInterface table = htablePool().getTable(name);
|
||||
try {
|
||||
table.put(p);
|
||||
} finally {
|
||||
htablePool().putTable(table);
|
||||
// table.close(); // XXX hbase 0.92
|
||||
}
|
||||
}
|
||||
public void put(Put p) throws IOException {
|
||||
HTableInterface table = htablePool().getTable(name);
|
||||
try {
|
||||
table.put(p);
|
||||
} finally {
|
||||
htablePool().putTable(table);
|
||||
// table.close(); // XXX hbase 0.92
|
||||
}
|
||||
}
|
||||
|
||||
public Result get(Get g) throws IOException {
|
||||
HTableInterface table = htablePool().getTable(name);
|
||||
try {
|
||||
return table.get(g);
|
||||
} finally {
|
||||
htablePool().putTable(table);
|
||||
// table.close(); // XXX hbase 0.92
|
||||
}
|
||||
}
|
||||
public Result get(Get g) throws IOException {
|
||||
HTableInterface table = htablePool().getTable(name);
|
||||
try {
|
||||
return table.get(g);
|
||||
} finally {
|
||||
htablePool().putTable(table);
|
||||
// table.close(); // XXX hbase 0.92
|
||||
}
|
||||
}
|
||||
|
||||
public HTableDescriptor getHtableDescriptor() throws IOException {
|
||||
HTableInterface table = htablePool().getTable(name);
|
||||
try {
|
||||
return table.getTableDescriptor();
|
||||
} finally {
|
||||
htablePool().putTable(table);
|
||||
}
|
||||
}
|
||||
public HTableDescriptor getHtableDescriptor() throws IOException {
|
||||
HTableInterface table = htablePool().getTable(name);
|
||||
try {
|
||||
return table.getTableDescriptor();
|
||||
} finally {
|
||||
htablePool().putTable(table);
|
||||
}
|
||||
}
|
||||
|
||||
protected boolean isRunning = false;
|
||||
protected boolean isRunning = false;
|
||||
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return isRunning;
|
||||
}
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return isRunning;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
try {
|
||||
if (getCreate()) {
|
||||
HBaseAdmin admin = hbase.admin();
|
||||
if (!admin.tableExists(name)) {
|
||||
HTableDescriptor desc = new HTableDescriptor(name);
|
||||
logger.info("hbase table '" + name + "' does not exist, creating it... " + desc);
|
||||
admin.createTable(desc);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.SEVERE, "problem creating hbase table " + name, e);
|
||||
}
|
||||
@Override
|
||||
public void start() {
|
||||
try {
|
||||
if (getCreate()) {
|
||||
HBaseAdmin admin = hbase.admin();
|
||||
if (!admin.tableExists(name)) {
|
||||
HTableDescriptor desc = new HTableDescriptor(name);
|
||||
logger.info("hbase table '" + name + "' does not exist, creating it... " + desc);
|
||||
admin.createTable(desc);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.log(Level.SEVERE, "problem creating hbase table " + name, e);
|
||||
}
|
||||
|
||||
isRunning = true;
|
||||
}
|
||||
isRunning = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
isRunning = false;
|
||||
// org.apache.hadoop.io.IOUtils.closeStream(htablePool); // XXX hbase 0.92
|
||||
if (htablePool != null) {
|
||||
try {
|
||||
htablePool.close();
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem closing HTablePool " + htablePool + " - " + e);
|
||||
}
|
||||
htablePool = null;
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public synchronized void stop() {
|
||||
isRunning = false;
|
||||
// org.apache.hadoop.io.IOUtils.closeStream(htablePool); // XXX hbase 0.92
|
||||
if (htablePool != null) {
|
||||
try {
|
||||
htablePool.close();
|
||||
} catch (IOException e) {
|
||||
logger.warning("problem closing HTablePool " + htablePool + " - " + e);
|
||||
}
|
||||
htablePool = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+86
-86
@@ -42,95 +42,95 @@ import org.archive.modules.recrawl.RecrawlAttributeConstants;
|
||||
* @contributor kenji
|
||||
*/
|
||||
public class MultiColumnRecrawlDataSchema extends RecrawlDataSchemaBase implements RecrawlDataSchema, RecrawlAttributeConstants {
|
||||
static final Logger logger = Logger.getLogger(MultiColumnRecrawlDataSchema.class.getName());
|
||||
static final Logger logger = Logger.getLogger(MultiColumnRecrawlDataSchema.class.getName());
|
||||
|
||||
public static final byte[] COLUMN_STATUS = Bytes.toBytes("s");
|
||||
public static final byte[] COLUMN_CONTENT_DIGEST = Bytes.toBytes("d");
|
||||
public static final byte[] COLUMN_ETAG = Bytes.toBytes("e");
|
||||
public static final byte[] COLUMN_LAST_MODIFIED = Bytes.toBytes("m");
|
||||
public static final byte[] COLUMN_STATUS = Bytes.toBytes("s");
|
||||
public static final byte[] COLUMN_CONTENT_DIGEST = Bytes.toBytes("d");
|
||||
public static final byte[] COLUMN_ETAG = Bytes.toBytes("e");
|
||||
public static final byte[] COLUMN_LAST_MODIFIED = Bytes.toBytes("m");
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.modules.hq.recrawl.RecrawlDataSchema#createPut()
|
||||
*/
|
||||
public Put createPut(CrawlURI uri) {
|
||||
byte[] uriBytes = Bytes.toBytes(uri.toString());
|
||||
byte[] key = uriBytes;
|
||||
Put p = new Put(key);
|
||||
String digest = uri.getContentDigestSchemeString();
|
||||
if (digest != null) {
|
||||
p.add(columnFamily, COLUMN_CONTENT_DIGEST, Bytes.toBytes(digest));
|
||||
}
|
||||
p.add(columnFamily, COLUMN_STATUS, Bytes.toBytes(Integer.toString(uri.getFetchStatus())));
|
||||
org.apache.commons.httpclient.HttpMethod method = uri.getHttpMethod();
|
||||
if (method != null) {
|
||||
String etag = getHeaderValue(method, RecrawlAttributeConstants.A_ETAG_HEADER);
|
||||
if (etag != null) {
|
||||
// Etqg is usually quoted
|
||||
if (etag.length() >= 2 && etag.charAt(0) == '"' && etag.charAt(etag.length() - 1) == '"')
|
||||
etag = etag.substring(1, etag.length() - 1);
|
||||
p.add(columnFamily, COLUMN_ETAG, Bytes.toBytes(etag));
|
||||
}
|
||||
String lastmod = getHeaderValue(method, RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER);
|
||||
if (lastmod != null) {
|
||||
long lastmod_sec = parseHttpDate(lastmod);
|
||||
if (lastmod_sec == 0) {
|
||||
try {
|
||||
lastmod_sec = uri.getFetchCompletedTime();
|
||||
} catch (NullPointerException ex) {
|
||||
logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine());
|
||||
}
|
||||
}
|
||||
if (lastmod_sec != 0)
|
||||
p.add(columnFamily, COLUMN_LAST_MODIFIED, Bytes.toBytes(lastmod_sec));
|
||||
} else {
|
||||
try {
|
||||
long completed = uri.getFetchCompletedTime();
|
||||
if (completed != 0)
|
||||
p.add(columnFamily, COLUMN_LAST_MODIFIED, Bytes.toBytes(completed));
|
||||
} catch (NullPointerException ex) {
|
||||
logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine());
|
||||
}
|
||||
}
|
||||
}
|
||||
return p;
|
||||
}
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.modules.hq.recrawl.RecrawlDataSchema#createPut()
|
||||
*/
|
||||
public Put createPut(CrawlURI uri) {
|
||||
byte[] uriBytes = Bytes.toBytes(uri.toString());
|
||||
byte[] key = uriBytes;
|
||||
Put p = new Put(key);
|
||||
String digest = uri.getContentDigestSchemeString();
|
||||
if (digest != null) {
|
||||
p.add(columnFamily, COLUMN_CONTENT_DIGEST, Bytes.toBytes(digest));
|
||||
}
|
||||
p.add(columnFamily, COLUMN_STATUS, Bytes.toBytes(Integer.toString(uri.getFetchStatus())));
|
||||
org.apache.commons.httpclient.HttpMethod method = uri.getHttpMethod();
|
||||
if (method != null) {
|
||||
String etag = getHeaderValue(method, RecrawlAttributeConstants.A_ETAG_HEADER);
|
||||
if (etag != null) {
|
||||
// Etqg is usually quoted
|
||||
if (etag.length() >= 2 && etag.charAt(0) == '"' && etag.charAt(etag.length() - 1) == '"')
|
||||
etag = etag.substring(1, etag.length() - 1);
|
||||
p.add(columnFamily, COLUMN_ETAG, Bytes.toBytes(etag));
|
||||
}
|
||||
String lastmod = getHeaderValue(method, RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER);
|
||||
if (lastmod != null) {
|
||||
long lastmod_sec = parseHttpDate(lastmod);
|
||||
if (lastmod_sec == 0) {
|
||||
try {
|
||||
lastmod_sec = uri.getFetchCompletedTime();
|
||||
} catch (NullPointerException ex) {
|
||||
logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine());
|
||||
}
|
||||
}
|
||||
if (lastmod_sec != 0)
|
||||
p.add(columnFamily, COLUMN_LAST_MODIFIED, Bytes.toBytes(lastmod_sec));
|
||||
} else {
|
||||
try {
|
||||
long completed = uri.getFetchCompletedTime();
|
||||
if (completed != 0)
|
||||
p.add(columnFamily, COLUMN_LAST_MODIFIED, Bytes.toBytes(completed));
|
||||
} catch (NullPointerException ex) {
|
||||
logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine());
|
||||
}
|
||||
}
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.modules.hq.recrawl.RecrawlDataSchema#load(java.util.Map, org.apache.hadoop.hbase.client.Result)
|
||||
*/
|
||||
public void load(Result result, CrawlURI curi) {
|
||||
// check for "do-not-crawl" flag - any non-empty data tells not to crawl this
|
||||
// URL.
|
||||
byte[] nocrawl = result.getValue(columnFamily, COLUMN_NOCRAWL);
|
||||
if (nocrawl != null && nocrawl.length > 0) {
|
||||
// fetch status set to S_DEEMED_CHAFF, because this do-not-crawl flag
|
||||
// is primarily intended for preventing crawler from stepping on traps.
|
||||
curi.setFetchStatus(FetchStatusCodes.S_DEEMED_CHAFF);
|
||||
curi.getAnnotations().add("nocrawl");
|
||||
return;
|
||||
}
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.modules.hq.recrawl.RecrawlDataSchema#load(java.util.Map, org.apache.hadoop.hbase.client.Result)
|
||||
*/
|
||||
public void load(Result result, CrawlURI curi) {
|
||||
// check for "do-not-crawl" flag - any non-empty data tells not to crawl this
|
||||
// URL.
|
||||
byte[] nocrawl = result.getValue(columnFamily, COLUMN_NOCRAWL);
|
||||
if (nocrawl != null && nocrawl.length > 0) {
|
||||
// fetch status set to S_DEEMED_CHAFF, because this do-not-crawl flag
|
||||
// is primarily intended for preventing crawler from stepping on traps.
|
||||
curi.setFetchStatus(FetchStatusCodes.S_DEEMED_CHAFF);
|
||||
curi.getAnnotations().add("nocrawl");
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, Object> history = getFetchHistory(curi);
|
||||
// FetchHTTP ignores history with status <= 0
|
||||
byte[] status = result.getValue(columnFamily, COLUMN_STATUS);
|
||||
if (status != null) {
|
||||
// Note that status is stored as integer text. It's typically three-chars
|
||||
// that is less than 4-byte integer bits.
|
||||
history.put(RecrawlAttributeConstants.A_STATUS, Integer.parseInt(Bytes.toString(status)));
|
||||
byte[] etag = result.getValue(columnFamily, COLUMN_ETAG);
|
||||
if (etag != null) {
|
||||
history.put(RecrawlAttributeConstants.A_ETAG_HEADER, Bytes.toString(etag));
|
||||
}
|
||||
byte[] lastmod = result.getValue(columnFamily, COLUMN_LAST_MODIFIED);
|
||||
if (lastmod != null) {
|
||||
long lastmod_sec = Bytes.toLong(lastmod);
|
||||
history.put(RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER, formatHttpDate(lastmod_sec));
|
||||
}
|
||||
byte[] digest = result.getValue(columnFamily, COLUMN_CONTENT_DIGEST);
|
||||
if (digest != null) {
|
||||
history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, Bytes.toString(digest));
|
||||
}
|
||||
}
|
||||
}
|
||||
Map<String, Object> history = getFetchHistory(curi);
|
||||
// FetchHTTP ignores history with status <= 0
|
||||
byte[] status = result.getValue(columnFamily, COLUMN_STATUS);
|
||||
if (status != null) {
|
||||
// Note that status is stored as integer text. It's typically three-chars
|
||||
// that is less than 4-byte integer bits.
|
||||
history.put(RecrawlAttributeConstants.A_STATUS, Integer.parseInt(Bytes.toString(status)));
|
||||
byte[] etag = result.getValue(columnFamily, COLUMN_ETAG);
|
||||
if (etag != null) {
|
||||
history.put(RecrawlAttributeConstants.A_ETAG_HEADER, Bytes.toString(etag));
|
||||
}
|
||||
byte[] lastmod = result.getValue(columnFamily, COLUMN_LAST_MODIFIED);
|
||||
if (lastmod != null) {
|
||||
long lastmod_sec = Bytes.toLong(lastmod);
|
||||
history.put(RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER, formatHttpDate(lastmod_sec));
|
||||
}
|
||||
byte[] digest = result.getValue(columnFamily, COLUMN_CONTENT_DIGEST);
|
||||
if (digest != null) {
|
||||
history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, Bytes.toString(digest));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import org.archive.modules.CrawlURI;
|
||||
* @contributor kenji
|
||||
*/
|
||||
public interface RecrawlDataSchema {
|
||||
public String getColumnFamily();
|
||||
public Put createPut(CrawlURI uri);
|
||||
public void load(Result result, CrawlURI curi);
|
||||
public String getColumnFamily();
|
||||
public Put createPut(CrawlURI uri);
|
||||
public void load(Result result, CrawlURI curi);
|
||||
}
|
||||
|
||||
+66
-66
@@ -25,81 +25,81 @@ import org.archive.modules.recrawl.RecrawlAttributeConstants;
|
||||
* @contributor kenji
|
||||
*/
|
||||
abstract public class RecrawlDataSchemaBase implements RecrawlDataSchema {
|
||||
private static final Logger logger = Logger.getLogger(RecrawlDataSchemaBase.class.getName());
|
||||
private static final Logger logger = Logger.getLogger(RecrawlDataSchemaBase.class.getName());
|
||||
|
||||
public static final byte[] DEFAULT_COLUMN_FAMILY = Bytes.toBytes("f");
|
||||
protected byte[] columnFamily = DEFAULT_COLUMN_FAMILY;
|
||||
public static final byte[] DEFAULT_COLUMN_FAMILY = Bytes.toBytes("f");
|
||||
protected byte[] columnFamily = DEFAULT_COLUMN_FAMILY;
|
||||
|
||||
public static final byte[] COLUMN_NOCRAWL = Bytes.toBytes("z");
|
||||
public static final byte[] COLUMN_NOCRAWL = Bytes.toBytes("z");
|
||||
|
||||
protected static String getHeaderValue(org.apache.commons.httpclient.HttpMethod method, String name) {
|
||||
org.apache.commons.httpclient.Header header = method.getResponseHeader(name);
|
||||
return header != null ? header.getValue() : null;
|
||||
}
|
||||
protected static String getHeaderValue(org.apache.commons.httpclient.HttpMethod method, String name) {
|
||||
org.apache.commons.httpclient.Header header = method.getResponseHeader(name);
|
||||
return header != null ? header.getValue() : null;
|
||||
}
|
||||
|
||||
public void setColumnFamily(String colf) {
|
||||
columnFamily = Bytes.toBytes(colf);
|
||||
}
|
||||
public void setColumnFamily(String colf) {
|
||||
columnFamily = Bytes.toBytes(colf);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getColumnFamily() {
|
||||
return Bytes.toString(columnFamily);
|
||||
}
|
||||
@Override
|
||||
public String getColumnFamily() {
|
||||
return Bytes.toString(columnFamily);
|
||||
}
|
||||
|
||||
/**
|
||||
* returns a Map to store recrawl data, which is properly stored in CrawlURI's
|
||||
* fetch history array property ({@link RecrawlAttributeConstants#A_FETCH_HISTORY} member of {@link CrawlURI#getData()}.)
|
||||
* if {@code uri} has no fetch history yet, it is first initialized with an array of length two so that
|
||||
* {@linkplain FetchHistoryProcessor} do not need to reallocate it (this only works for historyLength == 2, though).
|
||||
* @param uri CrawlURI from which fetch history is obtained.
|
||||
* @return Map object for storing re-crawl data (never null).
|
||||
* @see FetchHistoryProcessor
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
protected static Map<String, Object> getFetchHistory(CrawlURI uri) {
|
||||
Map<String, Object> data = uri.getData();
|
||||
Map<String, Object>[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY);
|
||||
if (history == null) {
|
||||
// only the first element is used by FetchHTTP, WarcWriterProcessor etc.
|
||||
// FetchHistoryProcessor casts history to HashMap[]. So it must be new HashMap[2], not Map[2]
|
||||
history = new HashMap[2];
|
||||
history[0] = new HashMap<String, Object>();
|
||||
// no need to set history[1]. it would simply be discarded by FetchHistoryProcessor.
|
||||
data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history);
|
||||
}
|
||||
return history[0];
|
||||
}
|
||||
/**
|
||||
* returns a Map to store recrawl data, which is properly stored in CrawlURI's
|
||||
* fetch history array property ({@link RecrawlAttributeConstants#A_FETCH_HISTORY} member of {@link CrawlURI#getData()}.)
|
||||
* if {@code uri} has no fetch history yet, it is first initialized with an array of length two so that
|
||||
* {@linkplain FetchHistoryProcessor} do not need to reallocate it (this only works for historyLength == 2, though).
|
||||
* @param uri CrawlURI from which fetch history is obtained.
|
||||
* @return Map object for storing re-crawl data (never null).
|
||||
* @see FetchHistoryProcessor
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
protected static Map<String, Object> getFetchHistory(CrawlURI uri) {
|
||||
Map<String, Object> data = uri.getData();
|
||||
Map<String, Object>[] history = (Map[])data.get(RecrawlAttributeConstants.A_FETCH_HISTORY);
|
||||
if (history == null) {
|
||||
// only the first element is used by FetchHTTP, WarcWriterProcessor etc.
|
||||
// FetchHistoryProcessor casts history to HashMap[]. So it must be new HashMap[2], not Map[2]
|
||||
history = new HashMap[2];
|
||||
history[0] = new HashMap<String, Object>();
|
||||
// no need to set history[1]. it would simply be discarded by FetchHistoryProcessor.
|
||||
data.put(RecrawlAttributeConstants.A_FETCH_HISTORY, history);
|
||||
}
|
||||
return history[0];
|
||||
}
|
||||
|
||||
public RecrawlDataSchemaBase() {
|
||||
super();
|
||||
}
|
||||
public RecrawlDataSchemaBase() {
|
||||
super();
|
||||
}
|
||||
|
||||
protected static final DateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
|
||||
protected static final DateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
|
||||
|
||||
/**
|
||||
* converts time in HTTP Date format {@code dateStr} to seconds
|
||||
* since epoch.
|
||||
* @param dateStr time in HTTP Date format.
|
||||
* @return seconds since epoch
|
||||
*/
|
||||
protected long parseHttpDate(String dateStr) {
|
||||
synchronized (HTTP_DATE_FORMAT) {
|
||||
try {
|
||||
Date d = HTTP_DATE_FORMAT.parse(dateStr);
|
||||
return d.getTime() / 1000;
|
||||
} catch (ParseException ex) {
|
||||
if (logger.isLoggable(Level.FINER))
|
||||
logger.fine("bad HTTP DATE: " + dateStr);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* converts time in HTTP Date format {@code dateStr} to seconds
|
||||
* since epoch.
|
||||
* @param dateStr time in HTTP Date format.
|
||||
* @return seconds since epoch
|
||||
*/
|
||||
protected long parseHttpDate(String dateStr) {
|
||||
synchronized (HTTP_DATE_FORMAT) {
|
||||
try {
|
||||
Date d = HTTP_DATE_FORMAT.parse(dateStr);
|
||||
return d.getTime() / 1000;
|
||||
} catch (ParseException ex) {
|
||||
if (logger.isLoggable(Level.FINER))
|
||||
logger.fine("bad HTTP DATE: " + dateStr);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected String formatHttpDate(long time) {
|
||||
synchronized (HTTP_DATE_FORMAT) {
|
||||
// format is not thread safe either
|
||||
return HTTP_DATE_FORMAT.format(new Date(time * 1000));
|
||||
}
|
||||
}
|
||||
protected String formatHttpDate(long time) {
|
||||
synchronized (HTTP_DATE_FORMAT) {
|
||||
// format is not thread safe either
|
||||
return HTTP_DATE_FORMAT.format(new Date(time * 1000));
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+112
-112
@@ -45,124 +45,124 @@ import org.json.JSONObject;
|
||||
*/
|
||||
public class SingleColumnJsonRecrawlDataSchema extends RecrawlDataSchemaBase
|
||||
implements RecrawlDataSchema {
|
||||
static final Logger logger = Logger.getLogger(SingleColumnJsonRecrawlDataSchema.class.getName());
|
||||
static final Logger logger = Logger.getLogger(SingleColumnJsonRecrawlDataSchema.class.getName());
|
||||
|
||||
public static byte[] DEFAULT_COLUMN = Bytes.toBytes("r");
|
||||
public static byte[] DEFAULT_COLUMN = Bytes.toBytes("r");
|
||||
|
||||
// JSON property names for re-crawl data properties
|
||||
public static final String PROPERTY_STATUS = "s";
|
||||
public static final String PROPERTY_CONTENT_DIGEST = "d";
|
||||
public static final String PROPERTY_ETAG = "e";
|
||||
public static final String PROPERTY_LAST_MODIFIED = "m";
|
||||
// JSON property names for re-crawl data properties
|
||||
public static final String PROPERTY_STATUS = "s";
|
||||
public static final String PROPERTY_CONTENT_DIGEST = "d";
|
||||
public static final String PROPERTY_ETAG = "e";
|
||||
public static final String PROPERTY_LAST_MODIFIED = "m";
|
||||
|
||||
// SHA1 scheme is assumed.
|
||||
public static final String CONTENT_DIGEST_SCHEME = "sha1:";
|
||||
// SHA1 scheme is assumed.
|
||||
public static final String CONTENT_DIGEST_SCHEME = "sha1:";
|
||||
|
||||
// single column for storing JSON of re-crawl data
|
||||
protected byte[] column = DEFAULT_COLUMN;
|
||||
public void setColumn(String column) {
|
||||
this.column = Bytes.toBytes(column);
|
||||
}
|
||||
public String getColumn() {
|
||||
return Bytes.toString(column);
|
||||
}
|
||||
// single column for storing JSON of re-crawl data
|
||||
protected byte[] column = DEFAULT_COLUMN;
|
||||
public void setColumn(String column) {
|
||||
this.column = Bytes.toBytes(column);
|
||||
}
|
||||
public String getColumn() {
|
||||
return Bytes.toString(column);
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.modules.hq.recrawl.RecrawlDataSchema#createPut(org.archive.modules.CrawlURI)
|
||||
*/
|
||||
public Put createPut(CrawlURI uri) {
|
||||
byte[] key = Bytes.toBytes(PersistProcessor.persistKeyFor(uri));
|
||||
Put p = new Put(key);
|
||||
JSONObject jo = new JSONObject();
|
||||
try {
|
||||
// TODO should we post warning message when scheme != "sha1"?
|
||||
String digest = uri.getContentDigestString();
|
||||
if (digest != null) {
|
||||
jo.put(PROPERTY_CONTENT_DIGEST, digest);
|
||||
}
|
||||
jo.put(PROPERTY_STATUS, uri.getFetchStatus());
|
||||
HttpMethod method = uri.getHttpMethod();
|
||||
if (method != null) {
|
||||
String etag = getHeaderValue(method, RecrawlAttributeConstants.A_ETAG_HEADER);
|
||||
if (etag != null) {
|
||||
// Etag is usually quoted
|
||||
if (etag.length() >= 2 && etag.charAt(0) == '"' && etag.charAt(etag.length() - 1) == '"')
|
||||
etag = etag.substring(1, etag.length() - 1);
|
||||
jo.put(PROPERTY_ETAG, etag);
|
||||
}
|
||||
String lastmod = getHeaderValue(method, RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER);
|
||||
if (lastmod != null) {
|
||||
// XXX value is not used anywhere, should it be?
|
||||
long lastmod_sec = parseHttpDate(lastmod);
|
||||
if (lastmod_sec == 0) {
|
||||
try {
|
||||
lastmod_sec = uri.getFetchCompletedTime();
|
||||
} catch (NullPointerException ex) {
|
||||
logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
long completed = uri.getFetchCompletedTime();
|
||||
if (completed != 0)
|
||||
jo.put(PROPERTY_LAST_MODIFIED, completed);
|
||||
} catch (NullPointerException ex) {
|
||||
logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (JSONException ex) {
|
||||
// should not happen - all values are either primitive or String.
|
||||
logger.log(Level.SEVERE, "JSON translation failed", ex);
|
||||
}
|
||||
p.add(columnFamily, column, Bytes.toBytes(jo.toString()));
|
||||
return p;
|
||||
}
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.modules.hq.recrawl.RecrawlDataSchema#createPut(org.archive.modules.CrawlURI)
|
||||
*/
|
||||
public Put createPut(CrawlURI uri) {
|
||||
byte[] key = Bytes.toBytes(PersistProcessor.persistKeyFor(uri));
|
||||
Put p = new Put(key);
|
||||
JSONObject jo = new JSONObject();
|
||||
try {
|
||||
// TODO should we post warning message when scheme != "sha1"?
|
||||
String digest = uri.getContentDigestString();
|
||||
if (digest != null) {
|
||||
jo.put(PROPERTY_CONTENT_DIGEST, digest);
|
||||
}
|
||||
jo.put(PROPERTY_STATUS, uri.getFetchStatus());
|
||||
HttpMethod method = uri.getHttpMethod();
|
||||
if (method != null) {
|
||||
String etag = getHeaderValue(method, RecrawlAttributeConstants.A_ETAG_HEADER);
|
||||
if (etag != null) {
|
||||
// Etag is usually quoted
|
||||
if (etag.length() >= 2 && etag.charAt(0) == '"' && etag.charAt(etag.length() - 1) == '"')
|
||||
etag = etag.substring(1, etag.length() - 1);
|
||||
jo.put(PROPERTY_ETAG, etag);
|
||||
}
|
||||
String lastmod = getHeaderValue(method, RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER);
|
||||
if (lastmod != null) {
|
||||
// XXX value is not used anywhere, should it be?
|
||||
long lastmod_sec = parseHttpDate(lastmod);
|
||||
if (lastmod_sec == 0) {
|
||||
try {
|
||||
lastmod_sec = uri.getFetchCompletedTime();
|
||||
} catch (NullPointerException ex) {
|
||||
logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
long completed = uri.getFetchCompletedTime();
|
||||
if (completed != 0)
|
||||
jo.put(PROPERTY_LAST_MODIFIED, completed);
|
||||
} catch (NullPointerException ex) {
|
||||
logger.warning("CrawlURI.getFetchCompletedTime():" + ex + " for " + uri.shortReportLine());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (JSONException ex) {
|
||||
// should not happen - all values are either primitive or String.
|
||||
logger.log(Level.SEVERE, "JSON translation failed", ex);
|
||||
}
|
||||
p.add(columnFamily, column, Bytes.toBytes(jo.toString()));
|
||||
return p;
|
||||
}
|
||||
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.modules.hq.recrawl.RecrawlDataSchema#load(org.apache.hadoop.hbase.client.Result)
|
||||
*/
|
||||
public void load(Result result, CrawlURI curi) {
|
||||
// check for "do-not-crawl" flag - any non-empty data tells not to crawl this
|
||||
// URL.
|
||||
byte[] nocrawl = result.getValue(columnFamily, COLUMN_NOCRAWL);
|
||||
if (nocrawl != null && nocrawl.length > 0) {
|
||||
// fetch status set to S_DEEMED_CHAFF, because this do-not-crawl flag
|
||||
// is primarily intended for preventing crawler from stepping on traps.
|
||||
curi.setFetchStatus(FetchStatusCodes.S_DEEMED_CHAFF);
|
||||
curi.getAnnotations().add("nocrawl");
|
||||
return;
|
||||
}
|
||||
/* (non-Javadoc)
|
||||
* @see org.archive.modules.hq.recrawl.RecrawlDataSchema#load(org.apache.hadoop.hbase.client.Result)
|
||||
*/
|
||||
public void load(Result result, CrawlURI curi) {
|
||||
// check for "do-not-crawl" flag - any non-empty data tells not to crawl this
|
||||
// URL.
|
||||
byte[] nocrawl = result.getValue(columnFamily, COLUMN_NOCRAWL);
|
||||
if (nocrawl != null && nocrawl.length > 0) {
|
||||
// fetch status set to S_DEEMED_CHAFF, because this do-not-crawl flag
|
||||
// is primarily intended for preventing crawler from stepping on traps.
|
||||
curi.setFetchStatus(FetchStatusCodes.S_DEEMED_CHAFF);
|
||||
curi.getAnnotations().add("nocrawl");
|
||||
return;
|
||||
}
|
||||
|
||||
Map<String, Object> history = getFetchHistory(curi);
|
||||
byte[] jsonBytes = result.getValue(columnFamily, column);
|
||||
if (jsonBytes != null) {
|
||||
JSONObject jo = null;
|
||||
try {
|
||||
jo = new JSONObject(Bytes.toString(jsonBytes));
|
||||
} catch (JSONException ex) {
|
||||
logger.warning(String.format("JSON parsing failed for key %1s: %2s",
|
||||
result.getRow(), ex.getMessage()));
|
||||
}
|
||||
if (jo != null) {
|
||||
int status = jo.optInt(PROPERTY_STATUS, -1);
|
||||
if (status >= 0) {
|
||||
history.put(RecrawlAttributeConstants.A_STATUS, status);
|
||||
}
|
||||
String digest = jo.optString(PROPERTY_CONTENT_DIGEST);
|
||||
if (digest != null) {
|
||||
history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, CONTENT_DIGEST_SCHEME + digest);
|
||||
}
|
||||
String etag = jo.optString(PROPERTY_ETAG);
|
||||
if (etag != null) {
|
||||
history.put(RecrawlAttributeConstants.A_ETAG_HEADER, etag);
|
||||
}
|
||||
long lastmod = jo.optLong(PROPERTY_LAST_MODIFIED);
|
||||
if (lastmod > 0) {
|
||||
history.put(RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER, formatHttpDate(lastmod));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Map<String, Object> history = getFetchHistory(curi);
|
||||
byte[] jsonBytes = result.getValue(columnFamily, column);
|
||||
if (jsonBytes != null) {
|
||||
JSONObject jo = null;
|
||||
try {
|
||||
jo = new JSONObject(Bytes.toString(jsonBytes));
|
||||
} catch (JSONException ex) {
|
||||
logger.warning(String.format("JSON parsing failed for key %1s: %2s",
|
||||
result.getRow(), ex.getMessage()));
|
||||
}
|
||||
if (jo != null) {
|
||||
int status = jo.optInt(PROPERTY_STATUS, -1);
|
||||
if (status >= 0) {
|
||||
history.put(RecrawlAttributeConstants.A_STATUS, status);
|
||||
}
|
||||
String digest = jo.optString(PROPERTY_CONTENT_DIGEST);
|
||||
if (digest != null) {
|
||||
history.put(RecrawlAttributeConstants.A_CONTENT_DIGEST, CONTENT_DIGEST_SCHEME + digest);
|
||||
}
|
||||
String etag = jo.optString(PROPERTY_ETAG);
|
||||
if (etag != null) {
|
||||
history.put(RecrawlAttributeConstants.A_ETAG_HEADER, etag);
|
||||
}
|
||||
long lastmod = jo.optLong(PROPERTY_LAST_MODIFIED);
|
||||
if (lastmod > 0) {
|
||||
history.put(RecrawlAttributeConstants.A_LAST_MODIFIED_HEADER, formatHttpDate(lastmod));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -7,48 +7,48 @@ import org.apache.log4j.spi.LoggingEvent;
|
||||
|
||||
public class OneLineSimpleLayout extends Layout {
|
||||
|
||||
private OneLineSimpleLogger logger = new OneLineSimpleLogger();
|
||||
|
||||
@Override
|
||||
public void activateOptions() {
|
||||
}
|
||||
private OneLineSimpleLogger logger = new OneLineSimpleLogger();
|
||||
|
||||
@Override
|
||||
public String format(LoggingEvent event) {
|
||||
java.util.logging.Level level = convertLevel(event.getLevel());
|
||||
|
||||
LogRecord logRecord = new LogRecord(level, event.getMessage().toString());
|
||||
logRecord.setLoggerName(event.getLoggerName());
|
||||
logRecord.setMillis(event.getTimeStamp());
|
||||
logRecord.setSourceClassName(event.getLoggerName());
|
||||
logRecord.setSourceMethodName(event.getLocationInformation().getMethodName());
|
||||
logRecord.setThreadID((int) Thread.currentThread().getId());
|
||||
|
||||
return logger.format(logRecord);
|
||||
}
|
||||
@Override
|
||||
public void activateOptions() {
|
||||
}
|
||||
|
||||
protected java.util.logging.Level convertLevel(org.apache.log4j.Level log4jLevel) {
|
||||
switch (log4jLevel.toInt()) {
|
||||
case org.apache.log4j.Level.TRACE_INT:
|
||||
return java.util.logging.Level.FINER;
|
||||
case org.apache.log4j.Level.DEBUG_INT:
|
||||
return java.util.logging.Level.FINE;
|
||||
case org.apache.log4j.Level.INFO_INT:
|
||||
return java.util.logging.Level.INFO;
|
||||
case org.apache.log4j.Level.WARN_INT:
|
||||
return java.util.logging.Level.WARNING;
|
||||
case org.apache.log4j.Level.ERROR_INT:
|
||||
return java.util.logging.Level.SEVERE;
|
||||
case org.apache.log4j.Level.FATAL_INT:
|
||||
return java.util.logging.Level.SEVERE;
|
||||
default:
|
||||
return java.util.logging.Level.ALL;
|
||||
}
|
||||
}
|
||||
@Override
|
||||
public String format(LoggingEvent event) {
|
||||
java.util.logging.Level level = convertLevel(event.getLevel());
|
||||
|
||||
@Override
|
||||
public boolean ignoresThrowable() {
|
||||
return true;
|
||||
}
|
||||
LogRecord logRecord = new LogRecord(level, event.getMessage().toString());
|
||||
logRecord.setLoggerName(event.getLoggerName());
|
||||
logRecord.setMillis(event.getTimeStamp());
|
||||
logRecord.setSourceClassName(event.getLoggerName());
|
||||
logRecord.setSourceMethodName(event.getLocationInformation().getMethodName());
|
||||
logRecord.setThreadID((int) Thread.currentThread().getId());
|
||||
|
||||
return logger.format(logRecord);
|
||||
}
|
||||
|
||||
protected java.util.logging.Level convertLevel(org.apache.log4j.Level log4jLevel) {
|
||||
switch (log4jLevel.toInt()) {
|
||||
case org.apache.log4j.Level.TRACE_INT:
|
||||
return java.util.logging.Level.FINER;
|
||||
case org.apache.log4j.Level.DEBUG_INT:
|
||||
return java.util.logging.Level.FINE;
|
||||
case org.apache.log4j.Level.INFO_INT:
|
||||
return java.util.logging.Level.INFO;
|
||||
case org.apache.log4j.Level.WARN_INT:
|
||||
return java.util.logging.Level.WARNING;
|
||||
case org.apache.log4j.Level.ERROR_INT:
|
||||
return java.util.logging.Level.SEVERE;
|
||||
case org.apache.log4j.Level.FATAL_INT:
|
||||
return java.util.logging.Level.SEVERE;
|
||||
default:
|
||||
return java.util.logging.Level.ALL;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean ignoresThrowable() {
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+68
-68
@@ -23,26 +23,26 @@ import org.archive.util.Recorder;
|
||||
|
||||
public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
|
||||
|
||||
protected static final String TEST_URI = "http://www.youtube.com/watch?v=_BFJN62hZp0";
|
||||
protected static final String TEST_RESOURCE_FILE_NAME = "ExtractorYoutubeFormatStream.txt";
|
||||
|
||||
protected static final String TEST_URI = "http://www.youtube.com/watch?v=_BFJN62hZp0";
|
||||
protected static final String TEST_RESOURCE_FILE_NAME = "ExtractorYoutubeFormatStream.txt";
|
||||
|
||||
protected static final String[] EXPECTED_OUTLINKS_ALL = {
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=44&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=98D0AC4D1B545DE6D5C531A5DB7902877511632A.5700D588C7659DE8A8621EC5110CB638D0EF4A39",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=43&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=2E2A7F8D5A61497159C2A2CFBB07F62B98062500.33826449E77B3A5F5FE40D028857064D7313D60A",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=18&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=9534F06E230AFD98894138B4A5D6DF75D11BC316.A488BE98A3EADBE84BA4A8BDA61E12EB6CA7BB85",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=5&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=CA92F947487449BAB3D360E565331F5BE50D2134.8EC1CB03A41B459227D84F34D2C2AB7BC2BA95A0",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=36&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=793FD335B7B7EE9B77B457DA951DAED704FFC363.1B6AC609D71AB3EE17BDA911E2761A87C79A080F",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=17&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=06294DA91D0E1FE3C7DF7AD9D133EC95939B8D70.731CE14A56E9C61E4ABE43110CE7515C5F42C66B"
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=44&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=98D0AC4D1B545DE6D5C531A5DB7902877511632A.5700D588C7659DE8A8621EC5110CB638D0EF4A39",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=43&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=2E2A7F8D5A61497159C2A2CFBB07F62B98062500.33826449E77B3A5F5FE40D028857064D7313D60A",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=18&mt=1370471490&source=youtube&mv=m&sver=3&ratebypass=yes&ms=au&sparams=cp%2Cid%2Cip%2Cipbits%2Citag%2Cratebypass%2Csource%2Cupn%2Cexpire&ipbits=8&expire=1370493270&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&id=fc114937ada1669d&upn=t-LMF5MC9BA&signature=9534F06E230AFD98894138B4A5D6DF75D11BC316.A488BE98A3EADBE84BA4A8BDA61E12EB6CA7BB85",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=5&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=CA92F947487449BAB3D360E565331F5BE50D2134.8EC1CB03A41B459227D84F34D2C2AB7BC2BA95A0",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=36&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=793FD335B7B7EE9B77B457DA951DAED704FFC363.1B6AC609D71AB3EE17BDA911E2761A87C79A080F",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=17&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=06294DA91D0E1FE3C7DF7AD9D133EC95939B8D70.731CE14A56E9C61E4ABE43110CE7515C5F42C66B"
|
||||
};
|
||||
|
||||
protected static final String[] EXPECTED_OUTLINKS_SUBSET = {
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21"
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=34&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=78D14935180C9DA87E1C562719525D0BB6BE21F9.9BC13425338E5DD6C255EA77136CC424830BCD21"
|
||||
};
|
||||
|
||||
|
||||
protected static final String[] EXPECTED_SINGLE_DEFAULT_OUTLINK = {
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703",
|
||||
"http://r3---sn-a5m7znek.c.youtube.com/videoplayback?ip=208.70.31.237&key=yt1&factor=1.25&newshard=yes&cp=U0hWRVRUUV9MSkNONl9MTlVDOjRtUl9JQzM2NENr&itag=35&sparams=algorithm%2Cburst%2Ccp%2Cfactor%2Cid%2Cip%2Cipbits%2Citag%2Csource%2Cupn%2Cexpire&source=youtube&mv=m&sver=3&fexp=900352%2C924605%2C928201%2C901208%2C929123%2C929121%2C929915%2C929906%2C925714%2C929919%2C929119%2C931202%2C928017%2C912512%2C912518%2C906906%2C904830%2C930807%2C919373%2C906836%2C933701%2C900816%2C912711%2C929606%2C910075&ms=au&algorithm=throttle-factor&id=fc114937ada1669d&expire=1370493270&burst=40&ipbits=8&upn=t-LMF5MC9BA&mt=1370471490&signature=A23283ED964AA8EF061249CCA6199EDDA6543FF2.89798F8181F2250FCFF24F19B2E59D880D054703",
|
||||
};
|
||||
|
||||
@Override
|
||||
@@ -52,71 +52,71 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
|
||||
e.setLoggerModule(ulm);
|
||||
return e;
|
||||
}
|
||||
|
||||
|
||||
public void testAllInItagPriority() throws Exception {
|
||||
CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME);
|
||||
|
||||
|
||||
List<String> itagPriorityList = Arrays.asList("44", "35", "43", "34", "18", "5", "36", "17");
|
||||
extractor().setItagPriority(itagPriorityList);
|
||||
extractor().setExtractLimit(10);
|
||||
|
||||
extractor.process(testUri);
|
||||
|
||||
Set<Link> expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL);
|
||||
Set<Link> expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL);
|
||||
assertEquals(expected, testUri.getOutLinks());
|
||||
}
|
||||
|
||||
public void testAllNoPriority() throws Exception {
|
||||
CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME);
|
||||
|
||||
|
||||
extractor().setExtractLimit(0);
|
||||
extractor.process(testUri);
|
||||
|
||||
Set<Link> expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL);
|
||||
Set<Link> expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_ALL);
|
||||
assertEquals(expected, testUri.getOutLinks());
|
||||
}
|
||||
|
||||
// test that only itags in the priority list are extracted, even though
|
||||
// extract limit is large
|
||||
// test that only itags in the priority list are extracted, even though
|
||||
// extract limit is large
|
||||
public void testOnlyInItagPriorityBigLimit() throws Exception {
|
||||
CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME);
|
||||
|
||||
|
||||
List<String> itagPriorityList = Arrays.asList("44", "35", "43");
|
||||
extractor().setItagPriority(itagPriorityList);
|
||||
extractor().setExtractLimit(10);
|
||||
|
||||
extractor.process(testUri);
|
||||
|
||||
assertEquals(3, testUri.getOutLinks().size());
|
||||
assertEquals(3, testUri.getOutLinks().size());
|
||||
}
|
||||
|
||||
// test that only itags in the priority list are extracted, even though
|
||||
// extract limit is unset
|
||||
// test that only itags in the priority list are extracted, even though
|
||||
// extract limit is unset
|
||||
public void testOnlyInItagPriorityNoLimit() throws Exception {
|
||||
CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME);
|
||||
|
||||
|
||||
List<String> itagPriorityList = Arrays.asList("44", "35", "43");
|
||||
extractor().setItagPriority(itagPriorityList);
|
||||
extractor().setExtractLimit(0);
|
||||
|
||||
extractor.process(testUri);
|
||||
|
||||
assertEquals(3, testUri.getOutLinks().size());
|
||||
assertEquals(3, testUri.getOutLinks().size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void testNoPriorityWithLimit() throws InterruptedException, URIException, UnsupportedEncodingException, IOException {
|
||||
CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME);
|
||||
|
||||
|
||||
extractor().setExtractLimit(4);
|
||||
|
||||
extractor.process(testUri);
|
||||
|
||||
assertEquals(4, testUri.getOutLinks().size());
|
||||
assertEquals(4, testUri.getOutLinks().size());
|
||||
}
|
||||
|
||||
|
||||
public void testDontExtract() throws URIException, UnsupportedEncodingException, IOException, InterruptedException {
|
||||
// not a youtube watch url so shouldProcess() will return false
|
||||
// not a youtube watch url so shouldProcess() will return false
|
||||
CrawlURI testUri = createTestUri("http://archive.org/watch?w=blah", TEST_RESOURCE_FILE_NAME);
|
||||
extractor.process(testUri);
|
||||
assertEquals(Collections.EMPTY_SET, testUri.getOutLinks());
|
||||
@@ -125,17 +125,17 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
|
||||
public void testPriority() throws Exception {
|
||||
CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME);
|
||||
|
||||
// 37, 24 are not in url_stream_map; 35 appears before 34 in there, but
|
||||
// with this list we should get 34
|
||||
// 37, 24 are not in url_stream_map; 35 appears before 34 in there, but
|
||||
// with this list we should get 34
|
||||
extractor().setItagPriority(Arrays.asList("37", "24", "34", "35"));
|
||||
extractor().setExtractLimit(1);
|
||||
|
||||
extractor.process(testUri);
|
||||
|
||||
Set<Link> expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_SUBSET);
|
||||
Set<Link> expected = makeLinkSet(testUri, EXPECTED_OUTLINKS_SUBSET);
|
||||
assertEquals(expected, testUri.getOutLinks());
|
||||
}
|
||||
|
||||
|
||||
public void testAlternatePage() throws Exception {
|
||||
CrawlURI testUri = createTestUri("http://www.youtube.com/watch?v=OyJ3CafAM1Q","ExtractorYoutubeFormatStream2.txt");
|
||||
|
||||
@@ -146,44 +146,44 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
|
||||
}
|
||||
public void testDefaultItag() throws URIException, UnsupportedEncodingException, IOException, InterruptedException {
|
||||
CrawlURI testUri = createTestUri(TEST_URI, TEST_RESOURCE_FILE_NAME);
|
||||
|
||||
|
||||
extractor().setExtractLimit(1);
|
||||
|
||||
|
||||
assertEquals(Collections.EMPTY_LIST, extractor().getItagPriority());
|
||||
|
||||
|
||||
extractor.process(testUri);
|
||||
|
||||
Set<Link> expected = makeLinkSet(testUri, EXPECTED_SINGLE_DEFAULT_OUTLINK);
|
||||
assertEquals(expected, testUri.getOutLinks());
|
||||
}
|
||||
|
||||
private Set<Link> makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException {
|
||||
HashSet<Link> linkSet = new HashSet<Link>();
|
||||
private Set<Link> makeLinkSet(CrawlURI sourceUri, String[] urlStrs) throws URIException {
|
||||
HashSet<Link> linkSet = new HashSet<Link>();
|
||||
for (String urlStr : urlStrs) {
|
||||
linkSet.add(new Link(sourceUri.getUURI(),
|
||||
linkSet.add(new Link(sourceUri.getUURI(),
|
||||
UURIFactory.getInstance(urlStr),
|
||||
HTMLLinkContext.EMBED_MISC, Hop.EMBED)
|
||||
);
|
||||
);
|
||||
}
|
||||
return linkSet;
|
||||
}
|
||||
return linkSet;
|
||||
}
|
||||
|
||||
private ExtractorYoutubeFormatStream extractor() {
|
||||
return (ExtractorYoutubeFormatStream)extractor;
|
||||
}
|
||||
|
||||
private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException,
|
||||
UnsupportedEncodingException, IOException {
|
||||
UURI testUuri = UURIFactory.getInstance(urlStr);
|
||||
private ExtractorYoutubeFormatStream extractor() {
|
||||
return (ExtractorYoutubeFormatStream)extractor;
|
||||
}
|
||||
|
||||
private CrawlURI createTestUri(String urlStr, String resourceFileName) throws URIException,
|
||||
UnsupportedEncodingException, IOException {
|
||||
UURI testUuri = UURIFactory.getInstance(urlStr);
|
||||
CrawlURI testUri = new CrawlURI(testUuri, null, null, LinkContext.NAVLINK_MISC);
|
||||
|
||||
InputStream is = ExtractorYoutubeFormatStreamTest.class.getClassLoader().getResourceAsStream(resourceFileName);
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
|
||||
StringBuilder content = new StringBuilder();
|
||||
String line = "";
|
||||
while ((line = reader.readLine()) != null) {
|
||||
content.append(line);
|
||||
}
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(is, "UTF-8"));
|
||||
StringBuilder content = new StringBuilder();
|
||||
String line = "";
|
||||
while ((line = reader.readLine()) != null) {
|
||||
content.append(line);
|
||||
}
|
||||
Recorder recorder = createRecorder(content.toString(), "UTF-8");
|
||||
IOUtils.closeQuietly(is);
|
||||
|
||||
@@ -191,22 +191,22 @@ public class ExtractorYoutubeFormatStreamTest extends ContentExtractorTestBase {
|
||||
testUri.setFetchStatus(200);
|
||||
testUri.setRecorder(recorder);
|
||||
testUri.setContentSize(content.length());
|
||||
return testUri;
|
||||
}
|
||||
return testUri;
|
||||
}
|
||||
}
|
||||
|
||||
class UnitTestUriLoggerModule implements UriErrorLoggerModule {
|
||||
final private static Logger LOGGER =
|
||||
Logger.getLogger(UnitTestUriLoggerModule.class.getName());
|
||||
Logger.getLogger(UnitTestUriLoggerModule.class.getName());
|
||||
|
||||
public void info(String info) {
|
||||
LOGGER.log(Level.INFO, info);
|
||||
System.out.println("INFO - "+info);
|
||||
LOGGER.log(Level.INFO, info);
|
||||
System.out.println("INFO - "+info);
|
||||
}
|
||||
public void fine(String info) {
|
||||
System.out.println("Fine - "+info);
|
||||
LOGGER.log(Level.FINE, info);
|
||||
|
||||
System.out.println("Fine - "+info);
|
||||
LOGGER.log(Level.FINE, info);
|
||||
|
||||
}
|
||||
public void logUriError(URIException e, UURI u, CharSequence l) {
|
||||
LOGGER.log(Level.INFO, u.toString(), e);
|
||||
@@ -217,6 +217,6 @@ class UnitTestUriLoggerModule implements UriErrorLoggerModule {
|
||||
return LOGGER;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user